2026-03-23T19:22:03.258 INFO:root:teuthology version: 1.2.4.dev6+g1c580df7a 2026-03-23T19:22:03.265 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-23T19:22:03.288 INFO:teuthology.run:Config: archive_path: /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508 branch: tentacle description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock} email: null first_in_suite: false flavor: default job_id: '3508' last_in_suite: false machine_type: vps meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode name: kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: tentacle ansible.cephlab: branch: main repo: https://github.com/kshtsk/ceph-cm-ansible.git skip_tags: nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs vars: logical_volumes: lv_1: scratch_dev: true size: 25%VG vg: vg_nvme lv_2: scratch_dev: true size: 25%VG vg: vg_nvme lv_3: scratch_dev: true size: 25%VG vg: vg_nvme lv_4: scratch_dev: true size: 25%VG vg: vg_nvme timezone: UTC volume_groups: vg_nvme: pvs: /dev/vdb,/dev/vdc,/dev/vdd,/dev/vde ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd mclock iops capacity threshold hdd: 49000 osd memory target: 1610612736 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: 70f8415b300f041766fa27faf7d5472699e32388 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} cephadm: cephadm_binary_url: https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm install: ceph: flavor: default sha1: 70f8415b300f041766fa27faf7d5472699e32388 extra_system_packages: deb: - python3-jmespath - python3-xmltodict - s3cmd rpm: - bzip2 - perl-Test-Harness - python3-jmespath - python3-xmltodict - s3cmd thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: tt-tentacle sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 owner: kyr priority: 1000 repo: https://github.com/ceph/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 3051 sha1: 70f8415b300f041766fa27faf7d5472699e32388 sleep_before_teardown: 0 subset: 1/128 suite: rbd suite_branch: tt-tentacle suite_path: /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa suite_relpath: qa suite_repo: https://github.com/kshtsk/ceph.git suite_sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 targets: vm04.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhg/2GSak8G0a+2w5RjKmf+3iY5nURPEtfBQShXyIoJgJqjg2pYSFw3UJqPnH/xW3ngd6idIBZJS/903A7dyd0= vm08.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYsXVZtZMfIMUf0VakwgvN+ByRF0wMwOF9RQ9ibQa2la0cEC5/9df3h1qwh+9fObltloTOt+jwhJr08L+Vb+D8= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror.sh env: CEPH_ARGS: '' RBD_IMAGE_FEATURES: layering,exclusive-lock RBD_MIRROR_CONFIG_KEY: '1' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: clyso-debian-13 teuthology_repo: https://github.com/clyso/teuthology teuthology_sha1: 1c580df7a9c7c2aadc272da296344fd99f27c444 timestamp: 2026-03-20_22:04:26 tube: vps user: kyr verbose: false worker_log: /home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871 2026-03-23T19:22:03.289 INFO:teuthology.run:suite_path is set to /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa; will attempt to use it 2026-03-23T19:22:03.289 INFO:teuthology.run:Found tasks at /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa/tasks 2026-03-23T19:22:03.289 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-03-23T19:22:03.289 INFO:teuthology.task.internal:Checking packages... 2026-03-23T19:22:03.289 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '70f8415b300f041766fa27faf7d5472699e32388' 2026-03-23T19:22:03.289 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-03-23T19:22:03.289 INFO:teuthology.packaging:ref: None 2026-03-23T19:22:03.289 INFO:teuthology.packaging:tag: None 2026-03-23T19:22:03.289 INFO:teuthology.packaging:branch: tentacle 2026-03-23T19:22:03.289 INFO:teuthology.packaging:sha1: 70f8415b300f041766fa27faf7d5472699e32388 2026-03-23T19:22:03.289 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=tentacle 2026-03-23T19:22:04.111 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-721.g5bb32787 2026-03-23T19:22:04.112 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-03-23T19:22:04.112 INFO:teuthology.task.internal:no buildpackages task found 2026-03-23T19:22:04.112 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-03-23T19:22:04.113 INFO:teuthology.task.internal:Saving configuration 2026-03-23T19:22:04.119 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-03-23T19:22:04.120 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-03-23T19:22:04.126 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm04.local', 'description': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-23 19:20:55.664698', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:04', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhg/2GSak8G0a+2w5RjKmf+3iY5nURPEtfBQShXyIoJgJqjg2pYSFw3UJqPnH/xW3ngd6idIBZJS/903A7dyd0='} 2026-03-23T19:22:04.130 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm08.local', 'description': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-23 19:20:55.665205', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:08', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYsXVZtZMfIMUf0VakwgvN+ByRF0wMwOF9RQ9ibQa2la0cEC5/9df3h1qwh+9fObltloTOt+jwhJr08L+Vb+D8='} 2026-03-23T19:22:04.130 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-03-23T19:22:04.131 INFO:teuthology.task.internal:roles: ubuntu@vm04.local - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-03-23T19:22:04.131 INFO:teuthology.task.internal:roles: ubuntu@vm08.local - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-03-23T19:22:04.131 INFO:teuthology.run_tasks:Running task console_log... 2026-03-23T19:22:04.135 DEBUG:teuthology.task.console_log:vm04 does not support IPMI; excluding 2026-03-23T19:22:04.140 DEBUG:teuthology.task.console_log:vm08 does not support IPMI; excluding 2026-03-23T19:22:04.140 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f6d0f41c4c0>, signals=[15]) 2026-03-23T19:22:04.140 INFO:teuthology.run_tasks:Running task internal.connect... 2026-03-23T19:22:04.141 INFO:teuthology.task.internal:Opening connections... 2026-03-23T19:22:04.141 DEBUG:teuthology.task.internal:connecting to ubuntu@vm04.local 2026-03-23T19:22:04.142 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm04.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-23T19:22:04.198 DEBUG:teuthology.task.internal:connecting to ubuntu@vm08.local 2026-03-23T19:22:04.198 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm08.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-23T19:22:04.255 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-03-23T19:22:04.256 DEBUG:teuthology.orchestra.run.vm04:> uname -m 2026-03-23T19:22:04.304 INFO:teuthology.orchestra.run.vm04.stdout:x86_64 2026-03-23T19:22:04.304 DEBUG:teuthology.orchestra.run.vm04:> cat /etc/os-release 2026-03-23T19:22:04.358 INFO:teuthology.orchestra.run.vm04.stdout:NAME="CentOS Stream" 2026-03-23T19:22:04.358 INFO:teuthology.orchestra.run.vm04.stdout:VERSION="9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:ID="centos" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:ID_LIKE="rhel fedora" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:VERSION_ID="9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:PLATFORM_ID="platform:el9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:ANSI_COLOR="0;31" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:LOGO="fedora-logo-icon" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:HOME_URL="https://centos.org/" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-23T19:22:04.359 INFO:teuthology.orchestra.run.vm04.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-23T19:22:04.359 INFO:teuthology.lock.ops:Updating vm04.local on lock server 2026-03-23T19:22:04.363 DEBUG:teuthology.orchestra.run.vm08:> uname -m 2026-03-23T19:22:04.375 INFO:teuthology.orchestra.run.vm08.stdout:x86_64 2026-03-23T19:22:04.376 DEBUG:teuthology.orchestra.run.vm08:> cat /etc/os-release 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:NAME="CentOS Stream" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:VERSION="9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:ID="centos" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:ID_LIKE="rhel fedora" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:VERSION_ID="9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:PLATFORM_ID="platform:el9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:ANSI_COLOR="0;31" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:LOGO="fedora-logo-icon" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:HOME_URL="https://centos.org/" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-23T19:22:04.428 INFO:teuthology.orchestra.run.vm08.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-23T19:22:04.428 INFO:teuthology.lock.ops:Updating vm08.local on lock server 2026-03-23T19:22:04.432 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-03-23T19:22:04.434 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-03-23T19:22:04.435 INFO:teuthology.task.internal:Checking for old test directory... 2026-03-23T19:22:04.435 DEBUG:teuthology.orchestra.run.vm04:> test '!' -e /home/ubuntu/cephtest 2026-03-23T19:22:04.437 DEBUG:teuthology.orchestra.run.vm08:> test '!' -e /home/ubuntu/cephtest 2026-03-23T19:22:04.481 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-03-23T19:22:04.482 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-03-23T19:22:04.482 DEBUG:teuthology.orchestra.run.vm04:> test -z $(ls -A /var/lib/ceph) 2026-03-23T19:22:04.492 DEBUG:teuthology.orchestra.run.vm08:> test -z $(ls -A /var/lib/ceph) 2026-03-23T19:22:04.505 INFO:teuthology.orchestra.run.vm04.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-23T19:22:04.533 INFO:teuthology.orchestra.run.vm08.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-23T19:22:04.534 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-03-23T19:22:04.541 DEBUG:teuthology.orchestra.run.vm04:> test -e /ceph-qa-ready 2026-03-23T19:22:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:22:04.742 DEBUG:teuthology.orchestra.run.vm08:> test -e /ceph-qa-ready 2026-03-23T19:22:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:22:04.929 INFO:teuthology.run_tasks:Running task internal.base... 2026-03-23T19:22:04.931 INFO:teuthology.task.internal:Creating test directory... 2026-03-23T19:22:04.931 DEBUG:teuthology.orchestra.run.vm04:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-23T19:22:04.933 DEBUG:teuthology.orchestra.run.vm08:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-23T19:22:04.946 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-03-23T19:22:04.947 INFO:teuthology.run_tasks:Running task internal.archive... 2026-03-23T19:22:04.948 INFO:teuthology.task.internal:Creating archive directory... 2026-03-23T19:22:04.948 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-23T19:22:04.987 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-23T19:22:05.003 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-03-23T19:22:05.004 INFO:teuthology.task.internal:Enabling coredump saving... 2026-03-23T19:22:05.004 DEBUG:teuthology.orchestra.run.vm04:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-23T19:22:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:22:05.055 DEBUG:teuthology.orchestra.run.vm08:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-23T19:22:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:22:05.069 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-23T19:22:05.097 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-23T19:22:05.118 INFO:teuthology.orchestra.run.vm04.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-23T19:22:05.128 INFO:teuthology.orchestra.run.vm04.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-23T19:22:05.133 INFO:teuthology.orchestra.run.vm08.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-23T19:22:05.141 INFO:teuthology.orchestra.run.vm08.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-23T19:22:05.142 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-03-23T19:22:05.143 INFO:teuthology.task.internal:Configuring sudo... 2026-03-23T19:22:05.143 DEBUG:teuthology.orchestra.run.vm04:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-23T19:22:05.171 DEBUG:teuthology.orchestra.run.vm08:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-23T19:22:05.204 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-03-23T19:22:05.207 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-03-23T19:22:05.207 DEBUG:teuthology.orchestra.run.vm04:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-23T19:22:05.234 DEBUG:teuthology.orchestra.run.vm08:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-23T19:22:05.258 DEBUG:teuthology.orchestra.run.vm04:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-23T19:22:05.305 DEBUG:teuthology.orchestra.run.vm04:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-23T19:22:05.359 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:22:05.360 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-23T19:22:05.416 DEBUG:teuthology.orchestra.run.vm08:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-23T19:22:05.437 DEBUG:teuthology.orchestra.run.vm08:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-23T19:22:05.490 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:22:05.490 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-23T19:22:05.546 DEBUG:teuthology.orchestra.run.vm04:> sudo service rsyslog restart 2026-03-23T19:22:05.548 DEBUG:teuthology.orchestra.run.vm08:> sudo service rsyslog restart 2026-03-23T19:22:05.571 INFO:teuthology.orchestra.run.vm04.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-23T19:22:05.611 INFO:teuthology.orchestra.run.vm08.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-23T19:22:06.070 INFO:teuthology.run_tasks:Running task internal.timer... 2026-03-23T19:22:06.072 INFO:teuthology.task.internal:Starting timer... 2026-03-23T19:22:06.072 INFO:teuthology.run_tasks:Running task pcp... 2026-03-23T19:22:06.075 INFO:teuthology.run_tasks:Running task selinux... 2026-03-23T19:22:06.077 INFO:teuthology.task.selinux:Excluding vm04: VMs are not yet supported 2026-03-23T19:22:06.077 INFO:teuthology.task.selinux:Excluding vm08: VMs are not yet supported 2026-03-23T19:22:06.077 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-03-23T19:22:06.077 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-03-23T19:22:06.077 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-03-23T19:22:06.077 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-03-23T19:22:06.078 DEBUG:teuthology.task:Applying overrides for task ansible.cephlab: {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}} 2026-03-23T19:22:06.079 DEBUG:teuthology.repo_utils:Setting repo remote to https://github.com/kshtsk/ceph-cm-ansible.git 2026-03-23T19:22:06.080 INFO:teuthology.repo_utils:Fetching github.com_kshtsk_ceph-cm-ansible_main from origin 2026-03-23T19:22:06.588 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main to origin/main 2026-03-23T19:22:06.594 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-03-23T19:22:06.595 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu", "logical_volumes": {"lv_1": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_2": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_3": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}, "lv_4": {"scratch_dev": true, "size": "25%VG", "vg": "vg_nvme"}}, "timezone": "UTC", "volume_groups": {"vg_nvme": {"pvs": "/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde"}}}' -i /tmp/teuth_ansible_inventory5lmxkwfa --limit vm04.local,vm08.local /home/teuthos/src/github.com_kshtsk_ceph-cm-ansible_main/cephlab.yml --skip-tags nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs 2026-03-23T19:24:00.298 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@vm04.local'), Remote(name='ubuntu@vm08.local')] 2026-03-23T19:24:00.299 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm04.local' 2026-03-23T19:24:00.299 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm04.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-23T19:24:00.361 DEBUG:teuthology.orchestra.run.vm04:> true 2026-03-23T19:24:00.440 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm04.local' 2026-03-23T19:24:00.440 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@vm08.local' 2026-03-23T19:24:00.440 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm08.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-23T19:24:00.498 DEBUG:teuthology.orchestra.run.vm08:> true 2026-03-23T19:24:00.585 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@vm08.local' 2026-03-23T19:24:00.585 INFO:teuthology.run_tasks:Running task clock... 2026-03-23T19:24:00.588 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-03-23T19:24:00.588 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-23T19:24:00.588 DEBUG:teuthology.orchestra.run.vm04:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-23T19:24:00.590 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-03-23T19:24:00.590 DEBUG:teuthology.orchestra.run.vm08:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-23T19:24:00.615 INFO:teuthology.orchestra.run.vm04.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-23T19:24:00.629 INFO:teuthology.orchestra.run.vm04.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-23T19:24:00.651 INFO:teuthology.orchestra.run.vm04.stderr:sudo: ntpd: command not found 2026-03-23T19:24:00.659 INFO:teuthology.orchestra.run.vm08.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-03-23T19:24:00.662 INFO:teuthology.orchestra.run.vm04.stdout:506 Cannot talk to daemon 2026-03-23T19:24:00.676 INFO:teuthology.orchestra.run.vm08.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-03-23T19:24:00.676 INFO:teuthology.orchestra.run.vm04.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-23T19:24:00.691 INFO:teuthology.orchestra.run.vm04.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-23T19:24:00.700 INFO:teuthology.orchestra.run.vm08.stderr:sudo: ntpd: command not found 2026-03-23T19:24:00.711 INFO:teuthology.orchestra.run.vm08.stdout:506 Cannot talk to daemon 2026-03-23T19:24:00.724 INFO:teuthology.orchestra.run.vm08.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-03-23T19:24:00.737 INFO:teuthology.orchestra.run.vm08.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-03-23T19:24:00.749 INFO:teuthology.orchestra.run.vm04.stderr:bash: line 1: ntpq: command not found 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:=============================================================================== 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:^? srv01-nc.securepod.org 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:^? ntp2.kernfusion.at 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:^? kronos.mailus.de 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.753 INFO:teuthology.orchestra.run.vm04.stdout:^? y.ns.gin.ntt.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.786 INFO:teuthology.orchestra.run.vm08.stderr:bash: line 1: ntpq: command not found 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:=============================================================================== 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:^? ntp2.kernfusion.at 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:^? y.ns.gin.ntt.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:^? kronos.mailus.de 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.789 INFO:teuthology.orchestra.run.vm08.stdout:^? srv01-nc.securepod.org 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-03-23T19:24:00.789 INFO:teuthology.run_tasks:Running task install... 2026-03-23T19:24:00.792 DEBUG:teuthology.task.install:project ceph 2026-03-23T19:24:00.792 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-23T19:24:00.792 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}} 2026-03-23T19:24:00.792 INFO:teuthology.task.install:Using flavor: default 2026-03-23T19:24:00.794 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-03-23T19:24:00.794 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-03-23T19:24:00.794 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-23T19:24:00.795 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-23T19:24:00.795 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'tag': None, 'wait_for_package': False} 2026-03-23T19:24:00.795 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-23T19:24:01.437 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-23T19:24:01.437 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-23T19:24:01.502 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/ 2026-03-23T19:24:01.502 INFO:teuthology.task.install.rpm:Package version is 20.2.0-712.g70f8415b 2026-03-23T19:24:01.977 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-23T19:24:01.977 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:24:01.977 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-23T19:24:01.993 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/tentacle/70f8415b300f041766fa27faf7d5472699e32388/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-03-23T19:24:01.993 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:24:01.993 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-03-23T19:24:02.006 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-23T19:24:02.006 DEBUG:teuthology.orchestra.run.vm08:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-23T19:24:02.016 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror, bzip2, perl-Test-Harness, python3-jmespath, python3-xmltodict, s3cmd on remote rpm x86_64 2026-03-23T19:24:02.016 DEBUG:teuthology.orchestra.run.vm04:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/70f8415b300f041766fa27faf7d5472699e32388/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-03-23T19:24:02.074 DEBUG:teuthology.orchestra.run.vm08:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-23T19:24:02.083 DEBUG:teuthology.orchestra.run.vm04:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-03-23T19:24:02.148 DEBUG:teuthology.orchestra.run.vm08:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-23T19:24:02.158 DEBUG:teuthology.orchestra.run.vm04:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-03-23T19:24:02.215 INFO:teuthology.orchestra.run.vm08.stdout:check_obsoletes = 1 2026-03-23T19:24:02.217 DEBUG:teuthology.orchestra.run.vm08:> sudo yum clean all 2026-03-23T19:24:02.220 INFO:teuthology.orchestra.run.vm04.stdout:check_obsoletes = 1 2026-03-23T19:24:02.222 DEBUG:teuthology.orchestra.run.vm04:> sudo yum clean all 2026-03-23T19:24:02.386 INFO:teuthology.orchestra.run.vm08.stdout:41 files removed 2026-03-23T19:24:02.393 INFO:teuthology.orchestra.run.vm04.stdout:41 files removed 2026-03-23T19:24:02.407 DEBUG:teuthology.orchestra.run.vm08:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd rbd-mirror bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-23T19:24:02.419 DEBUG:teuthology.orchestra.run.vm04:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd rbd-mirror bzip2 perl-Test-Harness python3-jmespath python3-xmltodict s3cmd 2026-03-23T19:24:03.804 INFO:teuthology.orchestra.run.vm04.stdout:ceph packages for x86_64 72 kB/s | 87 kB 00:01 2026-03-23T19:24:03.845 INFO:teuthology.orchestra.run.vm08.stdout:ceph packages for x86_64 68 kB/s | 87 kB 00:01 2026-03-23T19:24:04.879 INFO:teuthology.orchestra.run.vm08.stdout:ceph noarch packages 18 kB/s | 18 kB 00:01 2026-03-23T19:24:04.882 INFO:teuthology.orchestra.run.vm04.stdout:ceph noarch packages 17 kB/s | 18 kB 00:01 2026-03-23T19:24:05.817 INFO:teuthology.orchestra.run.vm08.stdout:ceph source packages 2.1 kB/s | 1.9 kB 00:00 2026-03-23T19:24:05.855 INFO:teuthology.orchestra.run.vm04.stdout:ceph source packages 2.0 kB/s | 1.9 kB 00:00 2026-03-23T19:24:06.843 INFO:teuthology.orchestra.run.vm04.stdout:CentOS Stream 9 - BaseOS 9.2 MB/s | 8.9 MB 00:00 2026-03-23T19:24:08.448 INFO:teuthology.orchestra.run.vm04.stdout:CentOS Stream 9 - AppStream 28 MB/s | 27 MB 00:00 2026-03-23T19:24:11.763 INFO:teuthology.orchestra.run.vm04.stdout:CentOS Stream 9 - CRB 15 MB/s | 8.0 MB 00:00 2026-03-23T19:24:13.218 INFO:teuthology.orchestra.run.vm04.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 20 kB 00:00 2026-03-23T19:24:13.665 INFO:teuthology.orchestra.run.vm04.stdout:Extra Packages for Enterprise Linux 55 MB/s | 20 MB 00:00 2026-03-23T19:24:17.963 INFO:teuthology.orchestra.run.vm04.stdout:lab-extras 65 kB/s | 50 kB 00:00 2026-03-23T19:24:19.243 INFO:teuthology.orchestra.run.vm04.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-23T19:24:19.244 INFO:teuthology.orchestra.run.vm04.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-23T19:24:19.274 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout:====================================================================================== 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout:====================================================================================== 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout:Installing: 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout:Upgrading: 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout:Installing dependencies: 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-23T19:24:19.279 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-23T19:24:19.280 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Installing weak dependencies: 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:====================================================================================== 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Install 136 Packages 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Upgrade 2 Packages 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Total download size: 267 M 2026-03-23T19:24:19.281 INFO:teuthology.orchestra.run.vm04.stdout:Downloading Packages: 2026-03-23T19:24:21.171 INFO:teuthology.orchestra.run.vm04.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 14 kB/s | 6.5 kB 00:00 2026-03-23T19:24:22.339 INFO:teuthology.orchestra.run.vm04.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 805 kB/s | 939 kB 00:01 2026-03-23T19:24:22.458 INFO:teuthology.orchestra.run.vm04.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 1.3 MB/s | 154 kB 00:00 2026-03-23T19:24:23.750 INFO:teuthology.orchestra.run.vm04.stdout:(4/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 1.8 MB/s | 2.3 MB 00:01 2026-03-23T19:24:24.222 INFO:teuthology.orchestra.run.vm04.stdout:(5/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 2.0 MB/s | 962 kB 00:00 2026-03-23T19:24:24.351 INFO:teuthology.orchestra.run.vm04.stdout:(6/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 1.6 MB/s | 5.9 MB 00:03 2026-03-23T19:24:25.649 INFO:teuthology.orchestra.run.vm04.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 3.5 MB/s | 5.0 MB 00:01 2026-03-23T19:24:26.906 INFO:teuthology.orchestra.run.vm04.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 3.8 MB/s | 24 MB 00:06 2026-03-23T19:24:27.019 INFO:teuthology.orchestra.run.vm04.stdout:(9/138): ceph-selinux-20.2.0-712.g70f8415b.el9. 222 kB/s | 25 kB 00:00 2026-03-23T19:24:27.109 INFO:teuthology.orchestra.run.vm04.stdout:(10/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86 6.2 MB/s | 17 MB 00:02 2026-03-23T19:24:27.228 INFO:teuthology.orchestra.run.vm04.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 291 kB/s | 34 kB 00:00 2026-03-23T19:24:27.347 INFO:teuthology.orchestra.run.vm04.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 203 kB/s | 24 kB 00:00 2026-03-23T19:24:27.478 INFO:teuthology.orchestra.run.vm04.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 6.4 MB/s | 866 kB 00:00 2026-03-23T19:24:27.600 INFO:teuthology.orchestra.run.vm04.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.3 MB/s | 164 kB 00:00 2026-03-23T19:24:27.722 INFO:teuthology.orchestra.run.vm04.stdout:(15/138): librados-devel-20.2.0-712.g70f8415b.e 1.0 MB/s | 126 kB 00:00 2026-03-23T19:24:27.846 INFO:teuthology.orchestra.run.vm04.stdout:(16/138): libradosstriper1-20.2.0-712.g70f8415b 2.0 MB/s | 250 kB 00:00 2026-03-23T19:24:28.462 INFO:teuthology.orchestra.run.vm04.stdout:(17/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 8.4 MB/s | 24 MB 00:02 2026-03-23T19:24:28.484 INFO:teuthology.orchestra.run.vm04.stdout:(18/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 10 MB/s | 6.4 MB 00:00 2026-03-23T19:24:28.580 INFO:teuthology.orchestra.run.vm04.stdout:(19/138): python3-ceph-argparse-20.2.0-712.g70f 382 kB/s | 45 kB 00:00 2026-03-23T19:24:28.609 INFO:teuthology.orchestra.run.vm04.stdout:(20/138): python3-ceph-common-20.2.0-712.g70f84 1.4 MB/s | 175 kB 00:00 2026-03-23T19:24:28.700 INFO:teuthology.orchestra.run.vm04.stdout:(21/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.3 MB/s | 163 kB 00:00 2026-03-23T19:24:28.732 INFO:teuthology.orchestra.run.vm04.stdout:(22/138): python3-rados-20.2.0-712.g70f8415b.el 2.6 MB/s | 324 kB 00:00 2026-03-23T19:24:28.822 INFO:teuthology.orchestra.run.vm04.stdout:(23/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 304 kB 00:00 2026-03-23T19:24:28.852 INFO:teuthology.orchestra.run.vm04.stdout:(24/138): python3-rgw-20.2.0-712.g70f8415b.el9. 827 kB/s | 99 kB 00:00 2026-03-23T19:24:28.942 INFO:teuthology.orchestra.run.vm04.stdout:(25/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 760 kB/s | 91 kB 00:00 2026-03-23T19:24:29.062 INFO:teuthology.orchestra.run.vm04.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.5 MB/s | 180 kB 00:00 2026-03-23T19:24:29.180 INFO:teuthology.orchestra.run.vm04.stdout:(27/138): ceph-grafana-dashboards-20.2.0-712.g7 366 kB/s | 43 kB 00:00 2026-03-23T19:24:29.300 INFO:teuthology.orchestra.run.vm04.stdout:(28/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.4 MB/s | 173 kB 00:00 2026-03-23T19:24:30.037 INFO:teuthology.orchestra.run.vm04.stdout:(29/138): ceph-mgr-dashboard-20.2.0-712.g70f841 14 MB/s | 11 MB 00:00 2026-03-23T19:24:30.082 INFO:teuthology.orchestra.run.vm04.stdout:(30/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 2.4 MB/s | 2.9 MB 00:01 2026-03-23T19:24:30.213 INFO:teuthology.orchestra.run.vm04.stdout:(31/138): ceph-mgr-modules-core-20.2.0-712.g70f 2.2 MB/s | 290 kB 00:00 2026-03-23T19:24:30.332 INFO:teuthology.orchestra.run.vm04.stdout:(32/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 423 kB/s | 50 kB 00:00 2026-03-23T19:24:30.450 INFO:teuthology.orchestra.run.vm04.stdout:(33/138): ceph-prometheus-alerts-20.2.0-712.g70 147 kB/s | 17 kB 00:00 2026-03-23T19:24:30.541 INFO:teuthology.orchestra.run.vm04.stdout:(34/138): ceph-mgr-diskprediction-local-20.2.0- 15 MB/s | 7.4 MB 00:00 2026-03-23T19:24:30.571 INFO:teuthology.orchestra.run.vm04.stdout:(35/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 298 kB 00:00 2026-03-23T19:24:30.689 INFO:teuthology.orchestra.run.vm04.stdout:(36/138): cephadm-20.2.0-712.g70f8415b.el9.noar 6.8 MB/s | 1.0 MB 00:00 2026-03-23T19:24:30.772 INFO:teuthology.orchestra.run.vm04.stdout:(37/138): bzip2-1.0.8-11.el9.x86_64.rpm 273 kB/s | 55 kB 00:00 2026-03-23T19:24:30.879 INFO:teuthology.orchestra.run.vm04.stdout:(38/138): fuse-2.9.9-17.el9.x86_64.rpm 748 kB/s | 80 kB 00:00 2026-03-23T19:24:30.930 INFO:teuthology.orchestra.run.vm04.stdout:(39/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 792 kB/s | 40 kB 00:00 2026-03-23T19:24:30.982 INFO:teuthology.orchestra.run.vm04.stdout:(40/138): libconfig-1.7.2-9.el9.x86_64.rpm 1.4 MB/s | 72 kB 00:00 2026-03-23T19:24:30.991 INFO:teuthology.orchestra.run.vm04.stdout:(41/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 1.1 MB/s | 351 kB 00:00 2026-03-23T19:24:31.045 INFO:teuthology.orchestra.run.vm04.stdout:(42/138): libquadmath-11.5.0-14.el9.x86_64.rpm 3.4 MB/s | 184 kB 00:00 2026-03-23T19:24:31.096 INFO:teuthology.orchestra.run.vm04.stdout:(43/138): mailcap-2.1.49-5.el9.noarch.rpm 654 kB/s | 33 kB 00:00 2026-03-23T19:24:31.137 INFO:teuthology.orchestra.run.vm04.stdout:(44/138): libgfortran-11.5.0-14.el9.x86_64.rpm 5.0 MB/s | 794 kB 00:00 2026-03-23T19:24:31.148 INFO:teuthology.orchestra.run.vm04.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 1.8 MB/s | 93 kB 00:00 2026-03-23T19:24:31.191 INFO:teuthology.orchestra.run.vm04.stdout:(46/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.6 MB/s | 253 kB 00:00 2026-03-23T19:24:31.244 INFO:teuthology.orchestra.run.vm04.stdout:(47/138): python3-ply-3.11-14.el9.noarch.rpm 2.0 MB/s | 106 kB 00:00 2026-03-23T19:24:31.297 INFO:teuthology.orchestra.run.vm04.stdout:(48/138): python3-pycparser-2.20-6.el9.noarch.r 2.5 MB/s | 135 kB 00:00 2026-03-23T19:24:31.303 INFO:teuthology.orchestra.run.vm04.stdout:(49/138): python3-cryptography-36.0.1-5.el9.x86 8.1 MB/s | 1.2 MB 00:00 2026-03-23T19:24:31.349 INFO:teuthology.orchestra.run.vm04.stdout:(50/138): python3-pyparsing-2.4.7-9.el9.noarch. 2.8 MB/s | 150 kB 00:00 2026-03-23T19:24:31.355 INFO:teuthology.orchestra.run.vm04.stdout:(51/138): python3-requests-2.25.1-10.el9.noarch 2.4 MB/s | 126 kB 00:00 2026-03-23T19:24:31.403 INFO:teuthology.orchestra.run.vm04.stdout:(52/138): python3-urllib3-1.26.5-7.el9.noarch.r 4.0 MB/s | 218 kB 00:00 2026-03-23T19:24:31.407 INFO:teuthology.orchestra.run.vm04.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 3.4 MB/s | 182 kB 00:00 2026-03-23T19:24:31.457 INFO:teuthology.orchestra.run.vm04.stdout:(54/138): zip-3.0-35.el9.x86_64.rpm 4.9 MB/s | 266 kB 00:00 2026-03-23T19:24:31.516 INFO:teuthology.orchestra.run.vm04.stdout:(55/138): boost-program-options-1.75.0-13.el9.x 957 kB/s | 104 kB 00:00 2026-03-23T19:24:31.544 INFO:teuthology.orchestra.run.vm04.stdout:(56/138): flexiblas-3.0.4-9.el9.x86_64.rpm 341 kB/s | 30 kB 00:00 2026-03-23T19:24:31.571 INFO:teuthology.orchestra.run.vm04.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 547 kB/s | 15 kB 00:00 2026-03-23T19:24:31.626 INFO:teuthology.orchestra.run.vm04.stdout:(58/138): libnbd-1.20.3-4.el9.x86_64.rpm 2.9 MB/s | 164 kB 00:00 2026-03-23T19:24:31.666 INFO:teuthology.orchestra.run.vm04.stdout:(59/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.9 MB/s | 160 kB 00:00 2026-03-23T19:24:31.699 INFO:teuthology.orchestra.run.vm04.stdout:(60/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.4 MB/s | 45 kB 00:00 2026-03-23T19:24:31.746 INFO:teuthology.orchestra.run.vm04.stdout:(61/138): librdkafka-1.6.1-102.el9.x86_64.rpm 14 MB/s | 662 kB 00:00 2026-03-23T19:24:31.772 INFO:teuthology.orchestra.run.vm04.stdout:(62/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 9.5 MB/s | 246 kB 00:00 2026-03-23T19:24:31.793 INFO:teuthology.orchestra.run.vm04.stdout:(63/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 11 MB/s | 3.0 MB 00:00 2026-03-23T19:24:31.798 INFO:teuthology.orchestra.run.vm04.stdout:(64/138): libxslt-1.1.34-12.el9.x86_64.rpm 8.6 MB/s | 233 kB 00:00 2026-03-23T19:24:31.823 INFO:teuthology.orchestra.run.vm04.stdout:(65/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 9.7 MB/s | 292 kB 00:00 2026-03-23T19:24:31.827 INFO:teuthology.orchestra.run.vm04.stdout:(66/138): lua-5.4.4-4.el9.x86_64.rpm 6.5 MB/s | 188 kB 00:00 2026-03-23T19:24:31.849 INFO:teuthology.orchestra.run.vm04.stdout:(67/138): openblas-0.3.29-1.el9.x86_64.rpm 1.6 MB/s | 42 kB 00:00 2026-03-23T19:24:31.906 INFO:teuthology.orchestra.run.vm04.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 469 kB/s | 26 kB 00:00 2026-03-23T19:24:31.940 INFO:teuthology.orchestra.run.vm04.stdout:(69/138): perl-Test-Harness-3.42-461.el9.noarch 8.5 MB/s | 295 kB 00:00 2026-03-23T19:24:31.984 INFO:teuthology.orchestra.run.vm04.stdout:(70/138): protobuf-3.14.0-17.el9.x86_64.rpm 23 MB/s | 1.0 MB 00:00 2026-03-23T19:24:32.103 INFO:teuthology.orchestra.run.vm04.stdout:(71/138): python3-babel-2.9.1-2.el9.noarch.rpm 50 MB/s | 6.0 MB 00:00 2026-03-23T19:24:32.117 INFO:teuthology.orchestra.run.vm04.stdout:(72/138): openblas-openmp-0.3.29-1.el9.x86_64.r 18 MB/s | 5.3 MB 00:00 2026-03-23T19:24:32.142 INFO:teuthology.orchestra.run.vm04.stdout:(73/138): python3-devel-3.9.25-3.el9.x86_64.rpm 6.3 MB/s | 244 kB 00:00 2026-03-23T19:24:32.144 INFO:teuthology.orchestra.run.vm04.stdout:(74/138): python3-jinja2-2.11.3-8.el9.noarch.rp 9.1 MB/s | 249 kB 00:00 2026-03-23T19:24:32.173 INFO:teuthology.orchestra.run.vm04.stdout:(75/138): python3-jmespath-1.0.1-1.el9.noarch.r 1.5 MB/s | 48 kB 00:00 2026-03-23T19:24:32.179 INFO:teuthology.orchestra.run.vm04.stdout:(76/138): python3-libstoragemgmt-1.10.1-1.el9.x 5.0 MB/s | 177 kB 00:00 2026-03-23T19:24:32.200 INFO:teuthology.orchestra.run.vm04.stdout:(77/138): python3-markupsafe-1.1.1-12.el9.x86_6 1.3 MB/s | 35 kB 00:00 2026-03-23T19:24:32.360 INFO:teuthology.orchestra.run.vm04.stdout:(78/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 34 MB/s | 6.1 MB 00:00 2026-03-23T19:24:32.379 INFO:teuthology.orchestra.run.vm04.stdout:(79/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 2.4 MB/s | 442 kB 00:00 2026-03-23T19:24:32.393 INFO:teuthology.orchestra.run.vm04.stdout:(80/138): python3-packaging-20.9-5.el9.noarch.r 2.3 MB/s | 77 kB 00:00 2026-03-23T19:24:32.409 INFO:teuthology.orchestra.run.vm04.stdout:(81/138): python3-protobuf-3.14.0-17.el9.noarch 8.7 MB/s | 267 kB 00:00 2026-03-23T19:24:32.425 INFO:teuthology.orchestra.run.vm04.stdout:(82/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 4.9 MB/s | 157 kB 00:00 2026-03-23T19:24:32.437 INFO:teuthology.orchestra.run.vm04.stdout:(83/138): python3-pyasn1-modules-0.4.8-7.el9.no 9.8 MB/s | 277 kB 00:00 2026-03-23T19:24:32.454 INFO:teuthology.orchestra.run.vm04.stdout:(84/138): python3-requests-oauthlib-1.3.0-12.el 1.8 MB/s | 54 kB 00:00 2026-03-23T19:24:32.482 INFO:teuthology.orchestra.run.vm04.stdout:(85/138): python3-toml-0.10.2-6.el9.noarch.rpm 1.4 MB/s | 42 kB 00:00 2026-03-23T19:24:32.520 INFO:teuthology.orchestra.run.vm04.stdout:(86/138): qatlib-25.08.0-2.el9.x86_64.rpm 6.3 MB/s | 240 kB 00:00 2026-03-23T19:24:32.549 INFO:teuthology.orchestra.run.vm04.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 1.3 MB/s | 37 kB 00:00 2026-03-23T19:24:32.572 INFO:teuthology.orchestra.run.vm04.stdout:(88/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 2.8 MB/s | 66 kB 00:00 2026-03-23T19:24:32.604 INFO:teuthology.orchestra.run.vm04.stdout:(89/138): socat-1.7.4.1-8.el9.x86_64.rpm 9.5 MB/s | 303 kB 00:00 2026-03-23T19:24:32.625 INFO:teuthology.orchestra.run.vm04.stdout:(90/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.9 MB/s | 64 kB 00:00 2026-03-23T19:24:32.886 INFO:teuthology.orchestra.run.vm04.stdout:(91/138): ceph-test-20.2.0-712.g70f8415b.el9.x8 14 MB/s | 84 MB 00:05 2026-03-23T19:24:33.027 INFO:teuthology.orchestra.run.vm04.stdout:(92/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 33 MB/s | 19 MB 00:00 2026-03-23T19:24:33.028 INFO:teuthology.orchestra.run.vm04.stdout:(93/138): lua-devel-5.4.4-4.el9.x86_64.rpm 55 kB/s | 22 kB 00:00 2026-03-23T19:24:33.039 INFO:teuthology.orchestra.run.vm04.stdout:(94/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 28 MB/s | 308 kB 00:00 2026-03-23T19:24:33.041 INFO:teuthology.orchestra.run.vm04.stdout:(95/138): grpc-data-1.46.7-10.el9.noarch.rpm 8.3 MB/s | 19 kB 00:00 2026-03-23T19:24:33.045 INFO:teuthology.orchestra.run.vm04.stdout:(96/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 31 MB/s | 551 kB 00:00 2026-03-23T19:24:33.048 INFO:teuthology.orchestra.run.vm04.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 6.7 MB/s | 25 kB 00:00 2026-03-23T19:24:33.053 INFO:teuthology.orchestra.run.vm04.stdout:(98/138): liboath-2.6.12-1.el9.x86_64.rpm 11 MB/s | 49 kB 00:00 2026-03-23T19:24:33.058 INFO:teuthology.orchestra.run.vm04.stdout:(99/138): libunwind-1.6.2-1.el9.x86_64.rpm 15 MB/s | 67 kB 00:00 2026-03-23T19:24:33.067 INFO:teuthology.orchestra.run.vm04.stdout:(100/138): luarocks-3.9.2-5.el9.noarch.rpm 17 MB/s | 151 kB 00:00 2026-03-23T19:24:33.105 INFO:teuthology.orchestra.run.vm04.stdout:(101/138): libarrow-9.0.0-15.el9.x86_64.rpm 70 MB/s | 4.4 MB 00:00 2026-03-23T19:24:33.112 INFO:teuthology.orchestra.run.vm04.stdout:(102/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 19 MB/s | 838 kB 00:00 2026-03-23T19:24:33.115 INFO:teuthology.orchestra.run.vm04.stdout:(103/138): python3-asyncssh-2.13.2-5.el9.noarch 56 MB/s | 548 kB 00:00 2026-03-23T19:24:33.115 INFO:teuthology.orchestra.run.vm04.stdout:(104/138): python3-autocommand-2.2.2-8.el9.noar 7.1 MB/s | 29 kB 00:00 2026-03-23T19:24:33.118 INFO:teuthology.orchestra.run.vm04.stdout:(105/138): python3-backports-tarfile-1.2.0-1.el 17 MB/s | 60 kB 00:00 2026-03-23T19:24:33.120 INFO:teuthology.orchestra.run.vm04.stdout:(106/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 10 MB/s | 43 kB 00:00 2026-03-23T19:24:33.121 INFO:teuthology.orchestra.run.vm04.stdout:(107/138): python3-cachetools-4.2.4-1.el9.noarc 11 MB/s | 32 kB 00:00 2026-03-23T19:24:33.122 INFO:teuthology.orchestra.run.vm04.stdout:(108/138): python3-certifi-2023.05.07-4.el9.noa 6.3 MB/s | 14 kB 00:00 2026-03-23T19:24:33.130 INFO:teuthology.orchestra.run.vm04.stdout:(109/138): python3-cherrypy-18.6.1-2.el9.noarch 45 MB/s | 358 kB 00:00 2026-03-23T19:24:33.131 INFO:teuthology.orchestra.run.vm04.stdout:(110/138): python3-cheroot-10.0.1-4.el9.noarch. 17 MB/s | 173 kB 00:00 2026-03-23T19:24:33.136 INFO:teuthology.orchestra.run.vm04.stdout:(111/138): python3-google-auth-2.45.0-1.el9.noa 43 MB/s | 254 kB 00:00 2026-03-23T19:24:33.141 INFO:teuthology.orchestra.run.vm04.stdout:(112/138): python3-grpcio-tools-1.46.7-10.el9.x 29 MB/s | 144 kB 00:00 2026-03-23T19:24:33.144 INFO:teuthology.orchestra.run.vm04.stdout:(113/138): python3-jaraco-8.2.1-3.el9.noarch.rp 3.5 MB/s | 11 kB 00:00 2026-03-23T19:24:33.147 INFO:teuthology.orchestra.run.vm04.stdout:(114/138): python3-jaraco-classes-3.2.1-5.el9.n 5.9 MB/s | 18 kB 00:00 2026-03-23T19:24:33.151 INFO:teuthology.orchestra.run.vm04.stdout:(115/138): python3-jaraco-collections-3.0.0-8.e 6.7 MB/s | 23 kB 00:00 2026-03-23T19:24:33.154 INFO:teuthology.orchestra.run.vm04.stdout:(116/138): python3-jaraco-context-6.0.1-3.el9.n 5.9 MB/s | 20 kB 00:00 2026-03-23T19:24:33.164 INFO:teuthology.orchestra.run.vm04.stdout:(117/138): python3-grpcio-1.46.7-10.el9.x86_64. 63 MB/s | 2.0 MB 00:00 2026-03-23T19:24:33.166 INFO:teuthology.orchestra.run.vm04.stdout:(118/138): python3-jaraco-functools-3.5.0-2.el9 1.7 MB/s | 19 kB 00:00 2026-03-23T19:24:33.167 INFO:teuthology.orchestra.run.vm04.stdout:(119/138): python3-jaraco-text-4.0.0-2.el9.noar 11 MB/s | 26 kB 00:00 2026-03-23T19:24:33.175 INFO:teuthology.orchestra.run.vm04.stdout:(120/138): python3-more-itertools-8.12.0-2.el9. 9.0 MB/s | 79 kB 00:00 2026-03-23T19:24:33.180 INFO:teuthology.orchestra.run.vm04.stdout:(121/138): python3-natsort-7.1.1-5.el9.noarch.r 12 MB/s | 58 kB 00:00 2026-03-23T19:24:33.185 INFO:teuthology.orchestra.run.vm04.stdout:(122/138): python3-kubernetes-26.1.0-3.el9.noar 53 MB/s | 1.0 MB 00:00 2026-03-23T19:24:33.186 INFO:teuthology.orchestra.run.vm04.stdout:(123/138): python3-portend-3.1.0-2.el9.noarch.r 3.2 MB/s | 16 kB 00:00 2026-03-23T19:24:33.189 INFO:teuthology.orchestra.run.vm04.stdout:(124/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 21 MB/s | 90 kB 00:00 2026-03-23T19:24:33.190 INFO:teuthology.orchestra.run.vm04.stdout:(125/138): python3-repoze-lru-0.7-16.el9.noarch 7.4 MB/s | 31 kB 00:00 2026-03-23T19:24:33.194 INFO:teuthology.orchestra.run.vm04.stdout:(126/138): python3-routes-2.5.1-5.el9.noarch.rp 37 MB/s | 188 kB 00:00 2026-03-23T19:24:33.196 INFO:teuthology.orchestra.run.vm04.stdout:(127/138): python3-rsa-4.9-2.el9.noarch.rpm 8.9 MB/s | 59 kB 00:00 2026-03-23T19:24:33.197 INFO:teuthology.orchestra.run.vm04.stdout:(128/138): python3-tempora-5.0.0-2.el9.noarch.r 14 MB/s | 36 kB 00:00 2026-03-23T19:24:33.200 INFO:teuthology.orchestra.run.vm04.stdout:(129/138): python3-typing-extensions-4.15.0-1.e 25 MB/s | 86 kB 00:00 2026-03-23T19:24:33.202 INFO:teuthology.orchestra.run.vm04.stdout:(130/138): python3-websocket-client-1.2.3-2.el9 21 MB/s | 90 kB 00:00 2026-03-23T19:24:33.204 INFO:teuthology.orchestra.run.vm04.stdout:(131/138): python3-xmltodict-0.12.0-15.el9.noar 6.5 MB/s | 22 kB 00:00 2026-03-23T19:24:33.204 INFO:teuthology.orchestra.run.vm04.stdout:(132/138): python3-zc-lockfile-2.0-10.el9.noarc 8.0 MB/s | 20 kB 00:00 2026-03-23T19:24:33.208 INFO:teuthology.orchestra.run.vm04.stdout:(133/138): re2-20211101-20.el9.x86_64.rpm 41 MB/s | 191 kB 00:00 2026-03-23T19:24:33.211 INFO:teuthology.orchestra.run.vm04.stdout:(134/138): s3cmd-2.4.0-1.el9.noarch.rpm 31 MB/s | 206 kB 00:00 2026-03-23T19:24:33.233 INFO:teuthology.orchestra.run.vm04.stdout:(135/138): thrift-0.15.0-4.el9.x86_64.rpm 65 MB/s | 1.6 MB 00:00 2026-03-23T19:24:33.476 INFO:teuthology.orchestra.run.vm04.stdout:(136/138): protobuf-compiler-3.14.0-17.el9.x86_ 1.4 MB/s | 862 kB 00:00 2026-03-23T19:24:34.166 INFO:teuthology.orchestra.run.vm04.stdout:(137/138): librados2-20.2.0-712.g70f8415b.el9.x 3.7 MB/s | 3.5 MB 00:00 2026-03-23T19:24:34.299 INFO:teuthology.orchestra.run.vm04.stdout:(138/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.7 MB/s | 2.8 MB 00:01 2026-03-23T19:24:34.303 INFO:teuthology.orchestra.run.vm04.stdout:-------------------------------------------------------------------------------- 2026-03-23T19:24:34.303 INFO:teuthology.orchestra.run.vm04.stdout:Total 18 MB/s | 267 MB 00:15 2026-03-23T19:24:34.846 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:24:34.911 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:24:34.912 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:24:35.891 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:24:35.892 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:24:36.947 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:24:36.955 INFO:teuthology.orchestra.run.vm04.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-23T19:24:36.958 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-23T19:24:36.969 INFO:teuthology.orchestra.run.vm04.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-23T19:24:37.136 INFO:teuthology.orchestra.run.vm04.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-23T19:24:37.139 INFO:teuthology.orchestra.run.vm04.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-23T19:24:37.169 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-23T19:24:37.179 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-23T19:24:37.183 INFO:teuthology.orchestra.run.vm04.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-23T19:24:37.187 INFO:teuthology.orchestra.run.vm04.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-23T19:24:37.190 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-23T19:24:37.194 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-23T19:24:37.330 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-23T19:24:37.332 INFO:teuthology.orchestra.run.vm04.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:24:37.352 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:24:37.353 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-23T19:24:37.376 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-23T19:24:37.378 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:24:37.391 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:24:37.426 INFO:teuthology.orchestra.run.vm04.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-23T19:24:37.451 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-23T19:24:37.462 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-23T19:24:37.468 INFO:teuthology.orchestra.run.vm04.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-23T19:24:37.472 INFO:teuthology.orchestra.run.vm04.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-23T19:24:37.477 INFO:teuthology.orchestra.run.vm04.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-23T19:24:37.504 INFO:teuthology.orchestra.run.vm04.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-23T19:24:37.520 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-23T19:24:37.525 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-23T19:24:37.531 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-23T19:24:37.535 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-23T19:24:37.571 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-23T19:24:37.577 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-23T19:24:37.580 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-23T19:24:37.581 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-23T19:24:37.633 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-23T19:24:37.634 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-23T19:24:37.656 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-23T19:24:37.669 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-23T19:24:37.678 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-23T19:24:37.705 INFO:teuthology.orchestra.run.vm04.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-23T19:24:37.711 INFO:teuthology.orchestra.run.vm04.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-23T19:24:37.719 INFO:teuthology.orchestra.run.vm04.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-23T19:24:37.775 INFO:teuthology.orchestra.run.vm04.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-23T19:24:37.792 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-23T19:24:37.810 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-23T19:24:37.816 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-23T19:24:37.825 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-23T19:24:37.832 INFO:teuthology.orchestra.run.vm04.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-23T19:24:37.836 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-23T19:24:37.852 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-23T19:24:37.858 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-23T19:24:37.865 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-23T19:24:37.878 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-23T19:24:37.890 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-23T19:24:37.896 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-23T19:24:37.905 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-23T19:24:37.951 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-23T19:24:38.315 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-23T19:24:38.330 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-23T19:24:38.336 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-23T19:24:38.343 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-23T19:24:38.348 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-23T19:24:38.355 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-23T19:24:38.359 INFO:teuthology.orchestra.run.vm04.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-23T19:24:38.361 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-23T19:24:38.390 INFO:teuthology.orchestra.run.vm04.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-23T19:24:38.442 INFO:teuthology.orchestra.run.vm04.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-23T19:24:38.455 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-23T19:24:38.463 INFO:teuthology.orchestra.run.vm04.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-23T19:24:38.468 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-23T19:24:38.475 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-23T19:24:38.481 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-23T19:24:38.489 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-23T19:24:38.495 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-23T19:24:38.527 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-23T19:24:38.540 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-23T19:24:38.547 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-23T19:24:38.557 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-23T19:24:38.598 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-23T19:24:38.860 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-23T19:24:38.890 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-23T19:24:38.894 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-23T19:24:38.898 INFO:teuthology.orchestra.run.vm04.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-23T19:24:38.957 INFO:teuthology.orchestra.run.vm04.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-23T19:24:38.960 INFO:teuthology.orchestra.run.vm04.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-23T19:24:38.982 INFO:teuthology.orchestra.run.vm04.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-23T19:24:39.357 INFO:teuthology.orchestra.run.vm04.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-23T19:24:39.445 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-23T19:24:40.207 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-23T19:24:40.235 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-23T19:24:40.241 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-23T19:24:40.245 INFO:teuthology.orchestra.run.vm04.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-23T19:24:40.252 INFO:teuthology.orchestra.run.vm04.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-23T19:24:40.564 INFO:teuthology.orchestra.run.vm04.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-23T19:24:40.566 INFO:teuthology.orchestra.run.vm04.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-23T19:24:40.589 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-23T19:24:40.590 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-23T19:24:41.817 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:24:41.823 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:24:41.845 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:24:41.858 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-23T19:24:41.868 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-23T19:24:41.886 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-23T19:24:41.908 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-23T19:24:42.003 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-23T19:24:42.018 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-23T19:24:42.050 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-23T19:24:42.092 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-23T19:24:42.156 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-23T19:24:42.166 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-23T19:24:42.173 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-23T19:24:42.178 INFO:teuthology.orchestra.run.vm04.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-23T19:24:42.183 INFO:teuthology.orchestra.run.vm04.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-23T19:24:42.185 INFO:teuthology.orchestra.run.vm04.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-23T19:24:42.205 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-23T19:24:42.531 INFO:teuthology.orchestra.run.vm04.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-23T19:24:42.538 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-23T19:24:42.586 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-23T19:24:42.586 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-23T19:24:42.586 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-23T19:24:42.586 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:42.592 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /sys 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /proc 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /mnt 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /var/tmp 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /home 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /root 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /tmp 2026-03-23T19:24:48.788 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:48.903 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-23T19:24:48.925 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:49.162 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-23T19:24:49.182 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:49.190 INFO:teuthology.orchestra.run.vm04.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-23T19:24:49.193 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-23T19:24:49.211 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:24:49.211 INFO:teuthology.orchestra.run.vm04.stdout:Creating group 'qat' with GID 994. 2026-03-23T19:24:49.211 INFO:teuthology.orchestra.run.vm04.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-23T19:24:49.211 INFO:teuthology.orchestra.run.vm04.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-23T19:24:49.211 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:49.222 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:24:49.247 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:24:49.247 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-23T19:24:49.247 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:49.268 INFO:teuthology.orchestra.run.vm04.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-23T19:24:49.293 INFO:teuthology.orchestra.run.vm04.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-23T19:24:49.364 INFO:teuthology.orchestra.run.vm04.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-23T19:24:49.369 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-23T19:24:49.382 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-23T19:24:49.382 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:49.382 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-23T19:24:49.382 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:50.127 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-23T19:24:50.150 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:50.219 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-23T19:24:50.222 INFO:teuthology.orchestra.run.vm04.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-23T19:24:50.231 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-23T19:24:50.257 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-23T19:24:50.260 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-23T19:24:51.513 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-23T19:24:51.524 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-23T19:24:52.037 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-23T19:24:52.039 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-23T19:24:52.101 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-23T19:24:52.151 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-23T19:24:52.154 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-23T19:24:52.177 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:52.192 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-23T19:24:52.204 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-23T19:24:52.249 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-23T19:24:53.420 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-23T19:24:53.424 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-23T19:24:53.453 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:53.466 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-23T19:24:53.487 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-23T19:24:53.487 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:53.487 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-23T19:24:53.487 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:53.633 INFO:teuthology.orchestra.run.vm04.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-23T19:24:53.656 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:57.880 INFO:teuthology.orchestra.run.vm04.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-23T19:24:57.887 INFO:teuthology.orchestra.run.vm04.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-23T19:24:57.893 INFO:teuthology.orchestra.run.vm04.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-23T19:24:57.904 INFO:teuthology.orchestra.run.vm04.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-23T19:24:57.922 INFO:teuthology.orchestra.run.vm04.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-23T19:24:57.929 INFO:teuthology.orchestra.run.vm04.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-23T19:24:57.933 INFO:teuthology.orchestra.run.vm04.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-23T19:24:57.933 INFO:teuthology.orchestra.run.vm04.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-23T19:24:57.948 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-23T19:24:57.948 INFO:teuthology.orchestra.run.vm04.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-23T19:24:59.680 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-23T19:24:59.681 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-23T19:24:59.682 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-23T19:24:59.683 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-23T19:24:59.683 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-23T19:24:59.683 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-23T19:24:59.683 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-23T19:24:59.684 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-23T19:24:59.685 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-23T19:24:59.686 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout:Upgraded: 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout:Installed: 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-23T19:24:59.783 INFO:teuthology.orchestra.run.vm04.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: lua-5.4.4-4.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-ply-3.11-14.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-23T19:24:59.784 INFO:teuthology.orchestra.run.vm04.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: re2-1:20211101-20.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: unzip-6.0-59.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: zip-3.0-35.el9.x86_64 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:24:59.785 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:24:59.878 DEBUG:teuthology.parallel:result is None 2026-03-23T19:29:15.937 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - BaseOS 29 kB/s | 8.9 MB 05:10 2026-03-23T19:29:18.294 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - AppStream 15 MB/s | 27 MB 00:01 2026-03-23T19:29:22.228 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - CRB 6.9 MB/s | 8.0 MB 00:01 2026-03-23T19:29:23.384 INFO:teuthology.orchestra.run.vm08.stdout:CentOS Stream 9 - Extras packages 76 kB/s | 20 kB 00:00 2026-03-23T19:29:24.071 INFO:teuthology.orchestra.run.vm08.stdout:Extra Packages for Enterprise Linux 34 MB/s | 20 MB 00:00 2026-03-23T19:29:29.030 INFO:teuthology.orchestra.run.vm08.stdout:lab-extras 49 kB/s | 50 kB 00:01 2026-03-23T19:29:30.493 INFO:teuthology.orchestra.run.vm08.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-23T19:29:30.494 INFO:teuthology.orchestra.run.vm08.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2026-03-23T19:29:30.531 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout:Installing: 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: bzip2 x86_64 1.0.8-11.el9 baseos 55 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.5 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.9 M 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 939 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 ceph 154 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 ceph 962 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 173 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 11 M 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 7.4 M 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 50 k 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 ceph 84 M 2026-03-23T19:29:30.537 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 298 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 1.0 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 34 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 866 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 ceph 126 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: perl-Test-Harness noarch 1:3.42-461.el9 appstream 295 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 ceph 163 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 ceph 324 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 304 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 ceph 99 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 91 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.9 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 180 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: s3cmd noarch 2.4.0-1.el9 epel 206 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout:Upgrading: 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 3.5 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.8 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout:Installing dependencies: 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 43 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 ceph 2.3 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 290 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 ceph 5.0 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 ceph 17 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 ceph-noarch 17 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 ceph 25 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: fuse x86_64 2.9.9-17.el9 baseos 80 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 24 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 ceph 164 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 250 k 2026-03-23T19:29:30.538 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 ceph 6.4 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: perl-Benchmark noarch 1.23-483.el9 appstream 26 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 ceph 45 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 ceph 175 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-03-23T19:29:30.539 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1 noarch 0.4.8-7.el9 appstream 157 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 appstream 277 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Installing weak dependencies: 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:====================================================================================== 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Install 136 Packages 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Upgrade 2 Packages 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Total download size: 267 M 2026-03-23T19:29:30.540 INFO:teuthology.orchestra.run.vm08.stdout:Downloading Packages: 2026-03-23T19:29:31.920 INFO:teuthology.orchestra.run.vm08.stdout:(1/138): ceph-20.2.0-712.g70f8415b.el9.x86_64.r 13 kB/s | 6.5 kB 00:00 2026-03-23T19:29:32.744 INFO:teuthology.orchestra.run.vm08.stdout:(2/138): ceph-fuse-20.2.0-712.g70f8415b.el9.x86 1.1 MB/s | 939 kB 00:00 2026-03-23T19:29:32.864 INFO:teuthology.orchestra.run.vm08.stdout:(3/138): ceph-immutable-object-cache-20.2.0-712 1.3 MB/s | 154 kB 00:00 2026-03-23T19:29:33.296 INFO:teuthology.orchestra.run.vm08.stdout:(4/138): ceph-base-20.2.0-712.g70f8415b.el9.x86 3.1 MB/s | 5.9 MB 00:01 2026-03-23T19:29:33.343 INFO:teuthology.orchestra.run.vm08.stdout:(5/138): ceph-mds-20.2.0-712.g70f8415b.el9.x86_ 4.9 MB/s | 2.3 MB 00:00 2026-03-23T19:29:33.528 INFO:teuthology.orchestra.run.vm08.stdout:(6/138): ceph-mgr-20.2.0-712.g70f8415b.el9.x86_ 4.1 MB/s | 962 kB 00:00 2026-03-23T19:29:34.070 INFO:teuthology.orchestra.run.vm08.stdout:(7/138): ceph-mon-20.2.0-712.g70f8415b.el9.x86_ 7.0 MB/s | 5.0 MB 00:00 2026-03-23T19:29:35.007 INFO:teuthology.orchestra.run.vm08.stdout:(8/138): ceph-common-20.2.0-712.g70f8415b.el9.x 6.6 MB/s | 24 MB 00:03 2026-03-23T19:29:35.091 INFO:teuthology.orchestra.run.vm08.stdout:(9/138): ceph-osd-20.2.0-712.g70f8415b.el9.x86_ 11 MB/s | 17 MB 00:01 2026-03-23T19:29:35.123 INFO:teuthology.orchestra.run.vm08.stdout:(10/138): ceph-selinux-20.2.0-712.g70f8415b.el9 216 kB/s | 25 kB 00:00 2026-03-23T19:29:35.240 INFO:teuthology.orchestra.run.vm08.stdout:(11/138): libcephfs-devel-20.2.0-712.g70f8415b. 294 kB/s | 34 kB 00:00 2026-03-23T19:29:35.357 INFO:teuthology.orchestra.run.vm08.stdout:(12/138): libcephfs-proxy2-20.2.0-712.g70f8415b 208 kB/s | 24 kB 00:00 2026-03-23T19:29:35.485 INFO:teuthology.orchestra.run.vm08.stdout:(13/138): libcephfs2-20.2.0-712.g70f8415b.el9.x 6.6 MB/s | 866 kB 00:00 2026-03-23T19:29:35.603 INFO:teuthology.orchestra.run.vm08.stdout:(14/138): libcephsqlite-20.2.0-712.g70f8415b.el 1.4 MB/s | 164 kB 00:00 2026-03-23T19:29:35.721 INFO:teuthology.orchestra.run.vm08.stdout:(15/138): librados-devel-20.2.0-712.g70f8415b.e 1.0 MB/s | 126 kB 00:00 2026-03-23T19:29:35.840 INFO:teuthology.orchestra.run.vm08.stdout:(16/138): libradosstriper1-20.2.0-712.g70f8415b 2.0 MB/s | 250 kB 00:00 2026-03-23T19:29:36.151 INFO:teuthology.orchestra.run.vm08.stdout:(17/138): ceph-radosgw-20.2.0-712.g70f8415b.el9 11 MB/s | 24 MB 00:02 2026-03-23T19:29:36.270 INFO:teuthology.orchestra.run.vm08.stdout:(18/138): python3-ceph-argparse-20.2.0-712.g70f 379 kB/s | 45 kB 00:00 2026-03-23T19:29:36.391 INFO:teuthology.orchestra.run.vm08.stdout:(19/138): python3-ceph-common-20.2.0-712.g70f84 1.4 MB/s | 175 kB 00:00 2026-03-23T19:29:36.488 INFO:teuthology.orchestra.run.vm08.stdout:(20/138): librgw2-20.2.0-712.g70f8415b.el9.x86_ 9.9 MB/s | 6.4 MB 00:00 2026-03-23T19:29:36.512 INFO:teuthology.orchestra.run.vm08.stdout:(21/138): python3-cephfs-20.2.0-712.g70f8415b.e 1.3 MB/s | 163 kB 00:00 2026-03-23T19:29:36.618 INFO:teuthology.orchestra.run.vm08.stdout:(22/138): python3-rados-20.2.0-712.g70f8415b.el 2.4 MB/s | 324 kB 00:00 2026-03-23T19:29:36.634 INFO:teuthology.orchestra.run.vm08.stdout:(23/138): python3-rbd-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 304 kB 00:00 2026-03-23T19:29:36.736 INFO:teuthology.orchestra.run.vm08.stdout:(24/138): python3-rgw-20.2.0-712.g70f8415b.el9. 839 kB/s | 99 kB 00:00 2026-03-23T19:29:36.754 INFO:teuthology.orchestra.run.vm08.stdout:(25/138): rbd-fuse-20.2.0-712.g70f8415b.el9.x86 763 kB/s | 91 kB 00:00 2026-03-23T19:29:36.889 INFO:teuthology.orchestra.run.vm08.stdout:(26/138): rbd-nbd-20.2.0-712.g70f8415b.el9.x86_ 1.3 MB/s | 180 kB 00:00 2026-03-23T19:29:37.008 INFO:teuthology.orchestra.run.vm08.stdout:(27/138): ceph-grafana-dashboards-20.2.0-712.g7 365 kB/s | 43 kB 00:00 2026-03-23T19:29:37.112 INFO:teuthology.orchestra.run.vm08.stdout:(28/138): rbd-mirror-20.2.0-712.g70f8415b.el9.x 7.8 MB/s | 2.9 MB 00:00 2026-03-23T19:29:37.128 INFO:teuthology.orchestra.run.vm08.stdout:(29/138): ceph-mgr-cephadm-20.2.0-712.g70f8415b 1.4 MB/s | 173 kB 00:00 2026-03-23T19:29:37.856 INFO:teuthology.orchestra.run.vm08.stdout:(30/138): ceph-mgr-diskprediction-local-20.2.0- 10 MB/s | 7.4 MB 00:00 2026-03-23T19:29:37.980 INFO:teuthology.orchestra.run.vm08.stdout:(31/138): ceph-mgr-modules-core-20.2.0-712.g70f 2.3 MB/s | 290 kB 00:00 2026-03-23T19:29:38.069 INFO:teuthology.orchestra.run.vm08.stdout:(32/138): ceph-mgr-dashboard-20.2.0-712.g70f841 11 MB/s | 11 MB 00:00 2026-03-23T19:29:38.099 INFO:teuthology.orchestra.run.vm08.stdout:(33/138): ceph-mgr-rook-20.2.0-712.g70f8415b.el 423 kB/s | 50 kB 00:00 2026-03-23T19:29:38.185 INFO:teuthology.orchestra.run.vm08.stdout:(34/138): ceph-prometheus-alerts-20.2.0-712.g70 150 kB/s | 17 kB 00:00 2026-03-23T19:29:38.220 INFO:teuthology.orchestra.run.vm08.stdout:(35/138): ceph-volume-20.2.0-712.g70f8415b.el9. 2.4 MB/s | 298 kB 00:00 2026-03-23T19:29:38.315 INFO:teuthology.orchestra.run.vm08.stdout:(36/138): cephadm-20.2.0-712.g70f8415b.el9.noar 7.7 MB/s | 1.0 MB 00:00 2026-03-23T19:29:38.338 INFO:teuthology.orchestra.run.vm08.stdout:(37/138): bzip2-1.0.8-11.el9.x86_64.rpm 467 kB/s | 55 kB 00:00 2026-03-23T19:29:38.395 INFO:teuthology.orchestra.run.vm08.stdout:(38/138): fuse-2.9.9-17.el9.x86_64.rpm 1.4 MB/s | 80 kB 00:00 2026-03-23T19:29:38.425 INFO:teuthology.orchestra.run.vm08.stdout:(39/138): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.3 MB/s | 40 kB 00:00 2026-03-23T19:29:38.455 INFO:teuthology.orchestra.run.vm08.stdout:(40/138): libconfig-1.7.2-9.el9.x86_64.rpm 2.4 MB/s | 72 kB 00:00 2026-03-23T19:29:38.511 INFO:teuthology.orchestra.run.vm08.stdout:(41/138): cryptsetup-2.8.1-3.el9.x86_64.rpm 1.8 MB/s | 351 kB 00:00 2026-03-23T19:29:38.542 INFO:teuthology.orchestra.run.vm08.stdout:(42/138): libquadmath-11.5.0-14.el9.x86_64.rpm 5.8 MB/s | 184 kB 00:00 2026-03-23T19:29:38.549 INFO:teuthology.orchestra.run.vm08.stdout:(43/138): libgfortran-11.5.0-14.el9.x86_64.rpm 8.3 MB/s | 794 kB 00:00 2026-03-23T19:29:38.572 INFO:teuthology.orchestra.run.vm08.stdout:(44/138): mailcap-2.1.49-5.el9.noarch.rpm 1.1 MB/s | 33 kB 00:00 2026-03-23T19:29:38.579 INFO:teuthology.orchestra.run.vm08.stdout:(45/138): pciutils-3.7.0-7.el9.x86_64.rpm 3.1 MB/s | 93 kB 00:00 2026-03-23T19:29:38.603 INFO:teuthology.orchestra.run.vm08.stdout:(46/138): python3-cffi-1.14.5-5.el9.x86_64.rpm 8.0 MB/s | 253 kB 00:00 2026-03-23T19:29:38.633 INFO:teuthology.orchestra.run.vm08.stdout:(47/138): python3-ply-3.11-14.el9.noarch.rpm 3.5 MB/s | 106 kB 00:00 2026-03-23T19:29:38.663 INFO:teuthology.orchestra.run.vm08.stdout:(48/138): python3-pycparser-2.20-6.el9.noarch.r 4.4 MB/s | 135 kB 00:00 2026-03-23T19:29:38.671 INFO:teuthology.orchestra.run.vm08.stdout:(49/138): python3-cryptography-36.0.1-5.el9.x86 14 MB/s | 1.2 MB 00:00 2026-03-23T19:29:38.694 INFO:teuthology.orchestra.run.vm08.stdout:(50/138): python3-pyparsing-2.4.7-9.el9.noarch. 4.8 MB/s | 150 kB 00:00 2026-03-23T19:29:38.701 INFO:teuthology.orchestra.run.vm08.stdout:(51/138): python3-requests-2.25.1-10.el9.noarch 4.1 MB/s | 126 kB 00:00 2026-03-23T19:29:38.725 INFO:teuthology.orchestra.run.vm08.stdout:(52/138): python3-urllib3-1.26.5-7.el9.noarch.r 6.9 MB/s | 218 kB 00:00 2026-03-23T19:29:38.732 INFO:teuthology.orchestra.run.vm08.stdout:(53/138): unzip-6.0-59.el9.x86_64.rpm 5.8 MB/s | 182 kB 00:00 2026-03-23T19:29:38.758 INFO:teuthology.orchestra.run.vm08.stdout:(54/138): zip-3.0-35.el9.x86_64.rpm 8.0 MB/s | 266 kB 00:00 2026-03-23T19:29:38.863 INFO:teuthology.orchestra.run.vm08.stdout:(55/138): flexiblas-3.0.4-9.el9.x86_64.rpm 283 kB/s | 30 kB 00:00 2026-03-23T19:29:38.868 INFO:teuthology.orchestra.run.vm08.stdout:(56/138): boost-program-options-1.75.0-13.el9.x 765 kB/s | 104 kB 00:00 2026-03-23T19:29:38.981 INFO:teuthology.orchestra.run.vm08.stdout:(57/138): flexiblas-openblas-openmp-3.0.4-9.el9 132 kB/s | 15 kB 00:00 2026-03-23T19:29:39.040 INFO:teuthology.orchestra.run.vm08.stdout:(58/138): libnbd-1.20.3-4.el9.x86_64.rpm 2.7 MB/s | 164 kB 00:00 2026-03-23T19:29:39.079 INFO:teuthology.orchestra.run.vm08.stdout:(59/138): libpmemobj-1.12.1-1.el9.x86_64.rpm 4.1 MB/s | 160 kB 00:00 2026-03-23T19:29:39.105 INFO:teuthology.orchestra.run.vm08.stdout:(60/138): flexiblas-netlib-3.0.4-9.el9.x86_64.r 12 MB/s | 3.0 MB 00:00 2026-03-23T19:29:39.109 INFO:teuthology.orchestra.run.vm08.stdout:(61/138): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.5 MB/s | 45 kB 00:00 2026-03-23T19:29:39.144 INFO:teuthology.orchestra.run.vm08.stdout:(62/138): librdkafka-1.6.1-102.el9.x86_64.rpm 17 MB/s | 662 kB 00:00 2026-03-23T19:29:39.155 INFO:teuthology.orchestra.run.vm08.stdout:(63/138): libstoragemgmt-1.10.1-1.el9.x86_64.rp 5.3 MB/s | 246 kB 00:00 2026-03-23T19:29:39.186 INFO:teuthology.orchestra.run.vm08.stdout:(64/138): libxslt-1.1.34-12.el9.x86_64.rpm 5.5 MB/s | 233 kB 00:00 2026-03-23T19:29:39.212 INFO:teuthology.orchestra.run.vm08.stdout:(65/138): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.0 MB/s | 292 kB 00:00 2026-03-23T19:29:39.216 INFO:teuthology.orchestra.run.vm08.stdout:(66/138): lua-5.4.4-4.el9.x86_64.rpm 6.3 MB/s | 188 kB 00:00 2026-03-23T19:29:39.240 INFO:teuthology.orchestra.run.vm08.stdout:(67/138): openblas-0.3.29-1.el9.x86_64.rpm 1.5 MB/s | 42 kB 00:00 2026-03-23T19:29:39.269 INFO:teuthology.orchestra.run.vm08.stdout:(68/138): perl-Benchmark-1.23-483.el9.noarch.rp 945 kB/s | 26 kB 00:00 2026-03-23T19:29:39.314 INFO:teuthology.orchestra.run.vm08.stdout:(69/138): perl-Test-Harness-3.42-461.el9.noarch 6.4 MB/s | 295 kB 00:00 2026-03-23T19:29:39.366 INFO:teuthology.orchestra.run.vm08.stdout:(70/138): openblas-openmp-0.3.29-1.el9.x86_64.r 35 MB/s | 5.3 MB 00:00 2026-03-23T19:29:39.466 INFO:teuthology.orchestra.run.vm08.stdout:(71/138): protobuf-3.14.0-17.el9.x86_64.rpm 6.6 MB/s | 1.0 MB 00:00 2026-03-23T19:29:39.519 INFO:teuthology.orchestra.run.vm08.stdout:(72/138): python3-devel-3.9.25-3.el9.x86_64.rpm 4.5 MB/s | 244 kB 00:00 2026-03-23T19:29:39.551 INFO:teuthology.orchestra.run.vm08.stdout:(73/138): python3-jinja2-2.11.3-8.el9.noarch.rp 7.7 MB/s | 249 kB 00:00 2026-03-23T19:29:39.606 INFO:teuthology.orchestra.run.vm08.stdout:(74/138): python3-jmespath-1.0.1-1.el9.noarch.r 880 kB/s | 48 kB 00:00 2026-03-23T19:29:39.665 INFO:teuthology.orchestra.run.vm08.stdout:(75/138): python3-babel-2.9.1-2.el9.noarch.rpm 20 MB/s | 6.0 MB 00:00 2026-03-23T19:29:39.666 INFO:teuthology.orchestra.run.vm08.stdout:(76/138): python3-libstoragemgmt-1.10.1-1.el9.x 2.9 MB/s | 177 kB 00:00 2026-03-23T19:29:39.692 INFO:teuthology.orchestra.run.vm08.stdout:(77/138): python3-markupsafe-1.1.1-12.el9.x86_6 1.3 MB/s | 35 kB 00:00 2026-03-23T19:29:39.754 INFO:teuthology.orchestra.run.vm08.stdout:(78/138): python3-numpy-f2py-1.23.5-2.el9.x86_6 7.0 MB/s | 442 kB 00:00 2026-03-23T19:29:39.782 INFO:teuthology.orchestra.run.vm08.stdout:(79/138): python3-packaging-20.9-5.el9.noarch.r 2.7 MB/s | 77 kB 00:00 2026-03-23T19:29:39.837 INFO:teuthology.orchestra.run.vm08.stdout:(80/138): python3-protobuf-3.14.0-17.el9.noarch 4.8 MB/s | 267 kB 00:00 2026-03-23T19:29:39.900 INFO:teuthology.orchestra.run.vm08.stdout:(81/138): python3-numpy-1.23.5-2.el9.x86_64.rpm 26 MB/s | 6.1 MB 00:00 2026-03-23T19:29:39.902 INFO:teuthology.orchestra.run.vm08.stdout:(82/138): python3-pyasn1-0.4.8-7.el9.noarch.rpm 2.4 MB/s | 157 kB 00:00 2026-03-23T19:29:39.930 INFO:teuthology.orchestra.run.vm08.stdout:(83/138): python3-requests-oauthlib-1.3.0-12.el 1.9 MB/s | 54 kB 00:00 2026-03-23T19:29:39.946 INFO:teuthology.orchestra.run.vm08.stdout:(84/138): python3-pyasn1-modules-0.4.8-7.el9.no 6.0 MB/s | 277 kB 00:00 2026-03-23T19:29:39.986 INFO:teuthology.orchestra.run.vm08.stdout:(85/138): python3-toml-0.10.2-6.el9.noarch.rpm 1.0 MB/s | 42 kB 00:00 2026-03-23T19:29:40.035 INFO:teuthology.orchestra.run.vm08.stdout:(86/138): qatlib-25.08.0-2.el9.x86_64.rpm 4.8 MB/s | 240 kB 00:00 2026-03-23T19:29:40.078 INFO:teuthology.orchestra.run.vm08.stdout:(87/138): qatlib-service-25.08.0-2.el9.x86_64.r 858 kB/s | 37 kB 00:00 2026-03-23T19:29:40.105 INFO:teuthology.orchestra.run.vm08.stdout:(88/138): qatzip-libs-1.3.1-1.el9.x86_64.rpm 2.4 MB/s | 66 kB 00:00 2026-03-23T19:29:40.137 INFO:teuthology.orchestra.run.vm08.stdout:(89/138): socat-1.7.4.1-8.el9.x86_64.rpm 9.3 MB/s | 303 kB 00:00 2026-03-23T19:29:40.166 INFO:teuthology.orchestra.run.vm08.stdout:(90/138): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.2 MB/s | 64 kB 00:00 2026-03-23T19:29:40.316 INFO:teuthology.orchestra.run.vm08.stdout:(91/138): python3-scipy-1.9.3-2.el9.x86_64.rpm 50 MB/s | 19 MB 00:00 2026-03-23T19:29:40.513 INFO:teuthology.orchestra.run.vm08.stdout:(92/138): lua-devel-5.4.4-4.el9.x86_64.rpm 64 kB/s | 22 kB 00:00 2026-03-23T19:29:40.530 INFO:teuthology.orchestra.run.vm08.stdout:(93/138): abseil-cpp-20211102.0-4.el9.x86_64.rp 33 MB/s | 551 kB 00:00 2026-03-23T19:29:40.539 INFO:teuthology.orchestra.run.vm08.stdout:(94/138): gperftools-libs-2.9.1-3.el9.x86_64.rp 35 MB/s | 308 kB 00:00 2026-03-23T19:29:40.541 INFO:teuthology.orchestra.run.vm08.stdout:(95/138): grpc-data-1.46.7-10.el9.noarch.rpm 8.4 MB/s | 19 kB 00:00 2026-03-23T19:29:40.604 INFO:teuthology.orchestra.run.vm08.stdout:(96/138): libarrow-9.0.0-15.el9.x86_64.rpm 72 MB/s | 4.4 MB 00:00 2026-03-23T19:29:40.608 INFO:teuthology.orchestra.run.vm08.stdout:(97/138): libarrow-doc-9.0.0-15.el9.noarch.rpm 6.4 MB/s | 25 kB 00:00 2026-03-23T19:29:40.612 INFO:teuthology.orchestra.run.vm08.stdout:(98/138): liboath-2.6.12-1.el9.x86_64.rpm 13 MB/s | 49 kB 00:00 2026-03-23T19:29:40.615 INFO:teuthology.orchestra.run.vm08.stdout:(99/138): libunwind-1.6.2-1.el9.x86_64.rpm 19 MB/s | 67 kB 00:00 2026-03-23T19:29:40.621 INFO:teuthology.orchestra.run.vm08.stdout:(100/138): luarocks-3.9.2-5.el9.noarch.rpm 28 MB/s | 151 kB 00:00 2026-03-23T19:29:40.639 INFO:teuthology.orchestra.run.vm08.stdout:(101/138): parquet-libs-9.0.0-15.el9.x86_64.rpm 46 MB/s | 838 kB 00:00 2026-03-23T19:29:40.759 INFO:teuthology.orchestra.run.vm08.stdout:(102/138): python3-asyncssh-2.13.2-5.el9.noarch 4.5 MB/s | 548 kB 00:00 2026-03-23T19:29:40.764 INFO:teuthology.orchestra.run.vm08.stdout:(103/138): protobuf-compiler-3.14.0-17.el9.x86_ 1.9 MB/s | 862 kB 00:00 2026-03-23T19:29:40.765 INFO:teuthology.orchestra.run.vm08.stdout:(104/138): python3-autocommand-2.2.2-8.el9.noar 5.9 MB/s | 29 kB 00:00 2026-03-23T19:29:40.768 INFO:teuthology.orchestra.run.vm08.stdout:(105/138): python3-bcrypt-3.2.2-1.el9.x86_64.rp 13 MB/s | 43 kB 00:00 2026-03-23T19:29:40.770 INFO:teuthology.orchestra.run.vm08.stdout:(106/138): python3-backports-tarfile-1.2.0-1.el 9.4 MB/s | 60 kB 00:00 2026-03-23T19:29:40.772 INFO:teuthology.orchestra.run.vm08.stdout:(107/138): python3-cachetools-4.2.4-1.el9.noarc 9.3 MB/s | 32 kB 00:00 2026-03-23T19:29:40.773 INFO:teuthology.orchestra.run.vm08.stdout:(108/138): python3-certifi-2023.05.07-4.el9.noa 6.6 MB/s | 14 kB 00:00 2026-03-23T19:29:40.777 INFO:teuthology.orchestra.run.vm08.stdout:(109/138): python3-cheroot-10.0.1-4.el9.noarch. 36 MB/s | 173 kB 00:00 2026-03-23T19:29:40.783 INFO:teuthology.orchestra.run.vm08.stdout:(110/138): python3-google-auth-2.45.0-1.el9.noa 44 MB/s | 254 kB 00:00 2026-03-23T19:29:40.792 INFO:teuthology.orchestra.run.vm08.stdout:(111/138): python3-cherrypy-18.6.1-2.el9.noarch 19 MB/s | 358 kB 00:00 2026-03-23T19:29:40.803 INFO:teuthology.orchestra.run.vm08.stdout:(112/138): python3-grpcio-tools-1.46.7-10.el9.x 14 MB/s | 144 kB 00:00 2026-03-23T19:29:40.807 INFO:teuthology.orchestra.run.vm08.stdout:(113/138): python3-jaraco-8.2.1-3.el9.noarch.rp 2.7 MB/s | 11 kB 00:00 2026-03-23T19:29:40.814 INFO:teuthology.orchestra.run.vm08.stdout:(114/138): python3-grpcio-1.46.7-10.el9.x86_64. 66 MB/s | 2.0 MB 00:00 2026-03-23T19:29:40.815 INFO:teuthology.orchestra.run.vm08.stdout:(115/138): python3-jaraco-classes-3.2.1-5.el9.n 2.2 MB/s | 18 kB 00:00 2026-03-23T19:29:40.817 INFO:teuthology.orchestra.run.vm08.stdout:(116/138): python3-jaraco-collections-3.0.0-8.e 9.5 MB/s | 23 kB 00:00 2026-03-23T19:29:40.818 INFO:teuthology.orchestra.run.vm08.stdout:(117/138): python3-jaraco-context-6.0.1-3.el9.n 6.0 MB/s | 20 kB 00:00 2026-03-23T19:29:40.819 INFO:teuthology.orchestra.run.vm08.stdout:(118/138): python3-jaraco-functools-3.5.0-2.el9 8.4 MB/s | 19 kB 00:00 2026-03-23T19:29:40.821 INFO:teuthology.orchestra.run.vm08.stdout:(119/138): python3-jaraco-text-4.0.0-2.el9.noar 11 MB/s | 26 kB 00:00 2026-03-23T19:29:40.828 INFO:teuthology.orchestra.run.vm08.stdout:(120/138): python3-more-itertools-8.12.0-2.el9. 11 MB/s | 79 kB 00:00 2026-03-23T19:29:40.836 INFO:teuthology.orchestra.run.vm08.stdout:(121/138): python3-kubernetes-26.1.0-3.el9.noar 64 MB/s | 1.0 MB 00:00 2026-03-23T19:29:40.837 INFO:teuthology.orchestra.run.vm08.stdout:(122/138): python3-natsort-7.1.1-5.el9.noarch.r 6.6 MB/s | 58 kB 00:00 2026-03-23T19:29:40.838 INFO:teuthology.orchestra.run.vm08.stdout:(123/138): python3-portend-3.1.0-2.el9.noarch.r 7.2 MB/s | 16 kB 00:00 2026-03-23T19:29:40.841 INFO:teuthology.orchestra.run.vm08.stdout:(124/138): python3-repoze-lru-0.7-16.el9.noarch 13 MB/s | 31 kB 00:00 2026-03-23T19:29:40.842 INFO:teuthology.orchestra.run.vm08.stdout:(125/138): python3-pyOpenSSL-21.0.0-1.el9.noarc 18 MB/s | 90 kB 00:00 2026-03-23T19:29:40.846 INFO:teuthology.orchestra.run.vm08.stdout:(126/138): python3-routes-2.5.1-5.el9.noarch.rp 36 MB/s | 188 kB 00:00 2026-03-23T19:29:40.857 INFO:teuthology.orchestra.run.vm08.stdout:(127/138): python3-rsa-4.9-2.el9.noarch.rpm 4.0 MB/s | 59 kB 00:00 2026-03-23T19:29:40.858 INFO:teuthology.orchestra.run.vm08.stdout:(128/138): python3-tempora-5.0.0-2.el9.noarch.r 3.0 MB/s | 36 kB 00:00 2026-03-23T19:29:40.862 INFO:teuthology.orchestra.run.vm08.stdout:(129/138): python3-typing-extensions-4.15.0-1.e 18 MB/s | 86 kB 00:00 2026-03-23T19:29:40.863 INFO:teuthology.orchestra.run.vm08.stdout:(130/138): python3-websocket-client-1.2.3-2.el9 17 MB/s | 90 kB 00:00 2026-03-23T19:29:40.865 INFO:teuthology.orchestra.run.vm08.stdout:(131/138): python3-xmltodict-0.12.0-15.el9.noar 9.0 MB/s | 22 kB 00:00 2026-03-23T19:29:40.866 INFO:teuthology.orchestra.run.vm08.stdout:(132/138): python3-zc-lockfile-2.0-10.el9.noarc 8.6 MB/s | 20 kB 00:00 2026-03-23T19:29:40.870 INFO:teuthology.orchestra.run.vm08.stdout:(133/138): re2-20211101-20.el9.x86_64.rpm 37 MB/s | 191 kB 00:00 2026-03-23T19:29:40.878 INFO:teuthology.orchestra.run.vm08.stdout:(134/138): s3cmd-2.4.0-1.el9.noarch.rpm 17 MB/s | 206 kB 00:00 2026-03-23T19:29:40.895 INFO:teuthology.orchestra.run.vm08.stdout:(135/138): thrift-0.15.0-4.el9.x86_64.rpm 65 MB/s | 1.6 MB 00:00 2026-03-23T19:29:41.941 INFO:teuthology.orchestra.run.vm08.stdout:(136/138): ceph-test-20.2.0-712.g70f8415b.el9.x 12 MB/s | 84 MB 00:06 2026-03-23T19:29:41.955 INFO:teuthology.orchestra.run.vm08.stdout:(137/138): librbd1-20.2.0-712.g70f8415b.el9.x86 2.7 MB/s | 2.8 MB 00:01 2026-03-23T19:29:42.036 INFO:teuthology.orchestra.run.vm08.stdout:(138/138): librados2-20.2.0-712.g70f8415b.el9.x 3.0 MB/s | 3.5 MB 00:01 2026-03-23T19:29:42.041 INFO:teuthology.orchestra.run.vm08.stdout:-------------------------------------------------------------------------------- 2026-03-23T19:29:42.041 INFO:teuthology.orchestra.run.vm08.stdout:Total 23 MB/s | 267 MB 00:11 2026-03-23T19:29:42.668 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:29:42.728 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:29:42.728 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:29:43.795 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:29:43.795 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:29:44.964 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:29:44.973 INFO:teuthology.orchestra.run.vm08.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/140 2026-03-23T19:29:44.975 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/140 2026-03-23T19:29:44.988 INFO:teuthology.orchestra.run.vm08.stdout: Installing : liboath-2.6.12-1.el9.x86_64 3/140 2026-03-23T19:29:45.171 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/140 2026-03-23T19:29:45.174 INFO:teuthology.orchestra.run.vm08.stdout: Upgrading : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-23T19:29:45.210 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 5/140 2026-03-23T19:29:45.222 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-23T19:29:45.233 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/140 2026-03-23T19:29:45.237 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/140 2026-03-23T19:29:45.242 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/140 2026-03-23T19:29:45.248 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 10/140 2026-03-23T19:29:45.396 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 11/140 2026-03-23T19:29:45.398 INFO:teuthology.orchestra.run.vm08.stdout: Upgrading : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:29:45.419 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:29:45.421 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-23T19:29:45.449 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 13/140 2026-03-23T19:29:45.451 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:29:45.467 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:29:45.507 INFO:teuthology.orchestra.run.vm08.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/140 2026-03-23T19:29:45.534 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/140 2026-03-23T19:29:45.548 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyasn1-0.4.8-7.el9.noarch 17/140 2026-03-23T19:29:45.558 INFO:teuthology.orchestra.run.vm08.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 18/140 2026-03-23T19:29:45.562 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lua-5.4.4-4.el9.x86_64 19/140 2026-03-23T19:29:45.568 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 20/140 2026-03-23T19:29:45.601 INFO:teuthology.orchestra.run.vm08.stdout: Installing : unzip-6.0-59.el9.x86_64 21/140 2026-03-23T19:29:45.620 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 22/140 2026-03-23T19:29:45.626 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 23/140 2026-03-23T19:29:45.634 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 24/140 2026-03-23T19:29:45.637 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 25/140 2026-03-23T19:29:45.678 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 26/140 2026-03-23T19:29:45.685 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 27/140 2026-03-23T19:29:45.688 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 28/140 2026-03-23T19:29:45.689 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-23T19:29:45.745 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 29/140 2026-03-23T19:29:45.747 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-23T19:29:45.770 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 30/140 2026-03-23T19:29:45.786 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 31/140 2026-03-23T19:29:45.796 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 32/140 2026-03-23T19:29:45.829 INFO:teuthology.orchestra.run.vm08.stdout: Installing : zip-3.0-35.el9.x86_64 33/140 2026-03-23T19:29:45.835 INFO:teuthology.orchestra.run.vm08.stdout: Installing : luarocks-3.9.2-5.el9.noarch 34/140 2026-03-23T19:29:45.845 INFO:teuthology.orchestra.run.vm08.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 35/140 2026-03-23T19:29:45.913 INFO:teuthology.orchestra.run.vm08.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 36/140 2026-03-23T19:29:45.933 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyasn1-modules-0.4.8-7.el9.noarch 37/140 2026-03-23T19:29:45.955 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rsa-4.9-2.el9.noarch 38/140 2026-03-23T19:29:45.964 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 39/140 2026-03-23T19:29:45.974 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 40/140 2026-03-23T19:29:45.982 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 41/140 2026-03-23T19:29:45.988 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 42/140 2026-03-23T19:29:46.007 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 43/140 2026-03-23T19:29:46.014 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 44/140 2026-03-23T19:29:46.021 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 45/140 2026-03-23T19:29:46.035 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 46/140 2026-03-23T19:29:46.047 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 47/140 2026-03-23T19:29:46.054 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 48/140 2026-03-23T19:29:46.067 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/140 2026-03-23T19:29:46.128 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/140 2026-03-23T19:29:46.570 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 51/140 2026-03-23T19:29:46.587 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 52/140 2026-03-23T19:29:46.594 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 53/140 2026-03-23T19:29:46.603 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 54/140 2026-03-23T19:29:46.609 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 55/140 2026-03-23T19:29:46.617 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 56/140 2026-03-23T19:29:46.621 INFO:teuthology.orchestra.run.vm08.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 57/140 2026-03-23T19:29:46.623 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 58/140 2026-03-23T19:29:46.657 INFO:teuthology.orchestra.run.vm08.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 59/140 2026-03-23T19:29:46.717 INFO:teuthology.orchestra.run.vm08.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 60/140 2026-03-23T19:29:46.736 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 61/140 2026-03-23T19:29:46.745 INFO:teuthology.orchestra.run.vm08.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 62/140 2026-03-23T19:29:46.751 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 63/140 2026-03-23T19:29:46.788 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 64/140 2026-03-23T19:29:46.793 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 65/140 2026-03-23T19:29:46.803 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 66/140 2026-03-23T19:29:46.808 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 67/140 2026-03-23T19:29:46.844 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 68/140 2026-03-23T19:29:46.860 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 69/140 2026-03-23T19:29:46.869 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 70/140 2026-03-23T19:29:46.881 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 71/140 2026-03-23T19:29:46.927 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 72/140 2026-03-23T19:29:47.239 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 73/140 2026-03-23T19:29:47.274 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 74/140 2026-03-23T19:29:47.279 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-23T19:29:47.284 INFO:teuthology.orchestra.run.vm08.stdout: Installing : perl-Benchmark-1.23-483.el9.noarch 76/140 2026-03-23T19:29:47.352 INFO:teuthology.orchestra.run.vm08.stdout: Installing : openblas-0.3.29-1.el9.x86_64 77/140 2026-03-23T19:29:47.355 INFO:teuthology.orchestra.run.vm08.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 78/140 2026-03-23T19:29:47.388 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 79/140 2026-03-23T19:29:47.822 INFO:teuthology.orchestra.run.vm08.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 80/140 2026-03-23T19:29:47.925 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 81/140 2026-03-23T19:29:48.768 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 82/140 2026-03-23T19:29:48.803 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 83/140 2026-03-23T19:29:48.809 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 84/140 2026-03-23T19:29:48.812 INFO:teuthology.orchestra.run.vm08.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 85/140 2026-03-23T19:29:48.820 INFO:teuthology.orchestra.run.vm08.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 86/140 2026-03-23T19:29:49.156 INFO:teuthology.orchestra.run.vm08.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 87/140 2026-03-23T19:29:49.157 INFO:teuthology.orchestra.run.vm08.stdout: Installing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-23T19:29:49.180 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 88/140 2026-03-23T19:29:49.181 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 89/140 2026-03-23T19:29:50.497 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:29:50.520 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:29:50.545 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 90/140 2026-03-23T19:29:50.561 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 91/140 2026-03-23T19:29:50.570 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-packaging-20.9-5.el9.noarch 92/140 2026-03-23T19:29:50.591 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-ply-3.11-14.el9.noarch 93/140 2026-03-23T19:29:50.613 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 94/140 2026-03-23T19:29:50.737 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 95/140 2026-03-23T19:29:50.757 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 96/140 2026-03-23T19:29:50.792 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 97/140 2026-03-23T19:29:50.837 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 98/140 2026-03-23T19:29:50.912 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 99/140 2026-03-23T19:29:50.923 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 100/140 2026-03-23T19:29:50.929 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 101/140 2026-03-23T19:29:50.935 INFO:teuthology.orchestra.run.vm08.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 102/140 2026-03-23T19:29:50.939 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 103/140 2026-03-23T19:29:50.941 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-23T19:29:50.959 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 104/140 2026-03-23T19:29:51.314 INFO:teuthology.orchestra.run.vm08.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 105/140 2026-03-23T19:29:51.321 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-23T19:29:51.365 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 106/140 2026-03-23T19:29:51.365 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-03-23T19:29:51.365 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-03-23T19:29:51.365 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:51.372 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-23T19:29:58.159 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 107/140 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /sys 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /proc 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /mnt 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /var/tmp 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /home 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /root 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /tmp 2026-03-23T19:29:58.165 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:58.440 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 108/140 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-03-23T19:29:58.465 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:58.721 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-23T19:29:58.745 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 109/140 2026-03-23T19:29:58.745 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:29:58.745 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-23T19:29:58.746 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-23T19:29:58.746 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-03-23T19:29:58.746 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:58.755 INFO:teuthology.orchestra.run.vm08.stdout: Installing : mailcap-2.1.49-5.el9.noarch 110/140 2026-03-23T19:29:58.759 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 111/140 2026-03-23T19:29:58.780 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:29:58.780 INFO:teuthology.orchestra.run.vm08.stdout:Creating group 'qat' with GID 994. 2026-03-23T19:29:58.780 INFO:teuthology.orchestra.run.vm08.stdout:Creating group 'libstoragemgmt' with GID 993. 2026-03-23T19:29:58.780 INFO:teuthology.orchestra.run.vm08.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 993 and GID 993. 2026-03-23T19:29:58.780 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:58.791 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:29:58.821 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 112/140 2026-03-23T19:29:58.821 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-03-23T19:29:58.821 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:58.843 INFO:teuthology.orchestra.run.vm08.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 113/140 2026-03-23T19:29:58.873 INFO:teuthology.orchestra.run.vm08.stdout: Installing : fuse-2.9.9-17.el9.x86_64 114/140 2026-03-23T19:29:58.955 INFO:teuthology.orchestra.run.vm08.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 115/140 2026-03-23T19:29:58.960 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-23T19:29:58.974 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 116/140 2026-03-23T19:29:58.974 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:29:58.974 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-23T19:29:58.974 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:59.797 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 117/140 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-03-23T19:29:59.824 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:29:59.899 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-23T19:29:59.902 INFO:teuthology.orchestra.run.vm08.stdout: Installing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 118/140 2026-03-23T19:29:59.912 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 119/140 2026-03-23T19:29:59.945 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 120/140 2026-03-23T19:29:59.948 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-23T19:30:01.326 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 121/140 2026-03-23T19:30:01.337 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-23T19:30:01.912 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 122/140 2026-03-23T19:30:01.915 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-23T19:30:01.992 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 123/140 2026-03-23T19:30:02.055 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 124/140 2026-03-23T19:30:02.069 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 125/140 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-03-23T19:30:02.091 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:02.107 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-23T19:30:02.119 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 126/140 2026-03-23T19:30:02.167 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 127/140 2026-03-23T19:30:03.422 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 128/140 2026-03-23T19:30:03.426 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 129/140 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-03-23T19:30:03.447 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:03.462 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-23T19:30:03.485 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f841 130/140 2026-03-23T19:30:03.485 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:30:03.485 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-23T19:30:03.485 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:03.634 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 131/140 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-03-23T19:30:03.655 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:08.207 INFO:teuthology.orchestra.run.vm08.stdout: Installing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 132/140 2026-03-23T19:30:08.216 INFO:teuthology.orchestra.run.vm08.stdout: Installing : perl-Test-Harness-1:3.42-461.el9.noarch 133/140 2026-03-23T19:30:08.223 INFO:teuthology.orchestra.run.vm08.stdout: Installing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 134/140 2026-03-23T19:30:08.236 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 135/140 2026-03-23T19:30:08.261 INFO:teuthology.orchestra.run.vm08.stdout: Installing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 136/140 2026-03-23T19:30:08.271 INFO:teuthology.orchestra.run.vm08.stdout: Installing : s3cmd-2.4.0-1.el9.noarch 137/140 2026-03-23T19:30:08.276 INFO:teuthology.orchestra.run.vm08.stdout: Installing : bzip2-1.0.8-11.el9.x86_64 138/140 2026-03-23T19:30:08.276 INFO:teuthology.orchestra.run.vm08.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-23T19:30:08.294 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 139/140 2026-03-23T19:30:08.294 INFO:teuthology.orchestra.run.vm08.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 4/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f841 5/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 6/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 7/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/140 2026-03-23T19:30:10.121 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 9/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 10/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 11/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 12/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_6 13/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_ 14/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 15/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 16/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 17/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_ 18/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 19/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9 20/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x 21/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 22/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 23/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 24/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 25/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 26/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 27/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 28/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.e 29/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noar 30/140 2026-03-23T19:30:10.122 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.no 31/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8 32/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9 33/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 34/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el 35/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 36/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 37/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : bzip2-1.0.8-11.el9.x86_64 38/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 39/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 40/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 41/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 42/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 43/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 44/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 45/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 46/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 47/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ply-3.11-14.el9.noarch 49/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 50/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 51/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 52/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 53/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : unzip-6.0-59.el9.x86_64 54/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : zip-3.0-35.el9.x86_64 55/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 56/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 57/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 58/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 59/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 60/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 61/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 62/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 63/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 64/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 65/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 66/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-5.4.4-4.el9.x86_64 67/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 68/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 69/140 2026-03-23T19:30:10.124 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : perl-Benchmark-1.23-483.el9.noarch 70/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : perl-Test-Harness-1:3.42-461.el9.noarch 71/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 72/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 73/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 74/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 75/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 76/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 77/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 78/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 79/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 80/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 81/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 82/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 83/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 84/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 86/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 87/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 88/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 89/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 90/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 91/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 92/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 93/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 94/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 95/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 96/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 97/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 98/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 99/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 100/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 101/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 102/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 103/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 104/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 105/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 106/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 107/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 108/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 109/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 110/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 111/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 112/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 113/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 114/140 2026-03-23T19:30:10.126 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 115/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 116/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 117/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 118/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 119/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 120/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 121/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 122/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 123/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 124/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 125/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 126/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 127/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 128/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 129/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 130/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 131/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 132/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 133/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : re2-1:20211101-20.el9.x86_64 134/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : s3cmd-2.4.0-1.el9.noarch 135/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 136/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 137/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 138/140 2026-03-23T19:30:10.127 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 139/140 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 140/140 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout:Upgraded: 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout:Installed: 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: bzip2-1.0.8-11.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.232 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: fuse-2.9.9-17.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: libxslt-1.1.34-12.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: lua-5.4.4-4.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: mailcap-2.1.49-5.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: perl-Benchmark-1.23-483.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: perl-Test-Harness-1:3.42-461.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-23T19:30:10.233 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply-3.11-14.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.234 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: re2-1:20211101-20.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: s3cmd-2.4.0-1.el9.noarch 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: socat-1.7.4.1-8.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: unzip-6.0-59.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: zip-3.0-35.el9.x86_64 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:10.235 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:30:10.340 DEBUG:teuthology.parallel:result is None 2026-03-23T19:30:10.340 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-23T19:30:10.965 DEBUG:teuthology.orchestra.run.vm04:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-23T19:30:10.988 INFO:teuthology.orchestra.run.vm04.stdout:20.2.0-712.g70f8415b.el9 2026-03-23T19:30:10.988 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-23T19:30:10.988 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-23T19:30:10.989 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=70f8415b300f041766fa27faf7d5472699e32388 2026-03-23T19:30:11.605 DEBUG:teuthology.orchestra.run.vm08:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-03-23T19:30:11.627 INFO:teuthology.orchestra.run.vm08.stdout:20.2.0-712.g70f8415b.el9 2026-03-23T19:30:11.627 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-712.g70f8415b.el9 2026-03-23T19:30:11.628 INFO:teuthology.task.install:The correct ceph version 20.2.0-712.g70f8415b is installed. 2026-03-23T19:30:11.629 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-03-23T19:30:11.629 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:11.629 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-23T19:30:11.659 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:11.659 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-03-23T19:30:11.696 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-03-23T19:30:11.696 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:11.696 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/usr/bin/daemon-helper 2026-03-23T19:30:11.729 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-23T19:30:11.796 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:11.796 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/daemon-helper 2026-03-23T19:30:11.824 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-03-23T19:30:11.889 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-03-23T19:30:11.889 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:11.889 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-23T19:30:11.915 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-23T19:30:11.984 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:11.984 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/adjust-ulimits 2026-03-23T19:30:12.014 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-03-23T19:30:12.084 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-03-23T19:30:12.085 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:12.085 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/usr/bin/stdin-killer 2026-03-23T19:30:12.114 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-23T19:30:12.180 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:12.180 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/usr/bin/stdin-killer 2026-03-23T19:30:12.206 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-03-23T19:30:12.271 INFO:teuthology.run_tasks:Running task ceph... 2026-03-23T19:30:12.310 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-03-23T19:30:12.310 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod 777 /var/log/ceph 2026-03-23T19:30:12.312 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 777 /var/log/ceph 2026-03-23T19:30:12.339 INFO:tasks.ceph:Disabling ceph logrotate... 2026-03-23T19:30:12.339 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-23T19:30:12.379 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f -- /etc/logrotate.d/ceph 2026-03-23T19:30:12.406 INFO:tasks.ceph:Creating extra log directories... 2026-03-23T19:30:12.406 DEBUG:teuthology.orchestra.run.vm04:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-23T19:30:12.447 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-03-23T19:30:12.483 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-03-23T19:30:12.483 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-23T19:30:12.483 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508', 'branch': 'tentacle', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '3508', 'last_in_suite': False, 'machine_type': 'vps', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3051, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'subset': '1/128', 'suite': 'rbd', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4', 'targets': {'vm04.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhg/2GSak8G0a+2w5RjKmf+3iY5nURPEtfBQShXyIoJgJqjg2pYSFw3UJqPnH/xW3ngd6idIBZJS/903A7dyd0=', 'vm08.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYsXVZtZMfIMUf0VakwgvN+ByRF0wMwOF9RQ9ibQa2la0cEC5/9df3h1qwh+9fObltloTOt+jwhJr08L+Vb+D8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_22:04:26', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871'} 2026-03-23T19:30:12.483 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-03-23T19:30:12.513 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-03-23T19:30:12.535 DEBUG:teuthology.orchestra.run.vm04:> sudo install -d -m0777 -- /var/run/ceph 2026-03-23T19:30:12.571 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/run/ceph 2026-03-23T19:30:12.608 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:12.608 DEBUG:teuthology.orchestra.run.vm04:> dd if=/scratch_devs of=/dev/stdout 2026-03-23T19:30:12.661 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-23T19:30:12.661 DEBUG:teuthology.orchestra.run.vm04:> stat /dev/vg_nvme/lv_1 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Device: 6h/6d Inode: 654 Links: 1 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Access: 2026-03-23 19:24:58.204393076 +0000 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Modify: 2026-03-23 19:23:27.198528346 +0000 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Change: 2026-03-23 19:23:27.198528346 +0000 2026-03-23T19:30:12.720 INFO:teuthology.orchestra.run.vm04.stdout: Birth: 2026-03-23 19:23:27.198528346 +0000 2026-03-23T19:30:12.721 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-23T19:30:12.789 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records in 2026-03-23T19:30:12.789 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records out 2026-03-23T19:30:12.790 INFO:teuthology.orchestra.run.vm04.stderr:512 bytes copied, 0.000103453 s, 4.9 MB/s 2026-03-23T19:30:12.791 DEBUG:teuthology.orchestra.run.vm04:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-23T19:30:12.850 DEBUG:teuthology.orchestra.run.vm04:> stat /dev/vg_nvme/lv_2 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Device: 6h/6d Inode: 685 Links: 1 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Access: 2026-03-23 19:24:58.204393076 +0000 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Modify: 2026-03-23 19:23:27.433528727 +0000 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Change: 2026-03-23 19:23:27.433528727 +0000 2026-03-23T19:30:12.907 INFO:teuthology.orchestra.run.vm04.stdout: Birth: 2026-03-23 19:23:27.433528727 +0000 2026-03-23T19:30:12.907 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-23T19:30:12.970 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records in 2026-03-23T19:30:12.971 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records out 2026-03-23T19:30:12.971 INFO:teuthology.orchestra.run.vm04.stderr:512 bytes copied, 0.000170128 s, 3.0 MB/s 2026-03-23T19:30:12.972 DEBUG:teuthology.orchestra.run.vm04:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-23T19:30:13.028 DEBUG:teuthology.orchestra.run.vm04:> stat /dev/vg_nvme/lv_3 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Device: 6h/6d Inode: 739 Links: 1 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Access: 2026-03-23 19:24:58.204393076 +0000 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Modify: 2026-03-23 19:23:27.666529105 +0000 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout:Change: 2026-03-23 19:23:27.666529105 +0000 2026-03-23T19:30:13.085 INFO:teuthology.orchestra.run.vm04.stdout: Birth: 2026-03-23 19:23:27.666529105 +0000 2026-03-23T19:30:13.085 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-23T19:30:13.149 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records in 2026-03-23T19:30:13.149 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records out 2026-03-23T19:30:13.149 INFO:teuthology.orchestra.run.vm04.stderr:512 bytes copied, 0.000146113 s, 3.5 MB/s 2026-03-23T19:30:13.150 DEBUG:teuthology.orchestra.run.vm04:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-23T19:30:13.207 DEBUG:teuthology.orchestra.run.vm04:> stat /dev/vg_nvme/lv_4 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Device: 6h/6d Inode: 775 Links: 1 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Access: 2026-03-23 19:24:58.205393077 +0000 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Modify: 2026-03-23 19:23:27.883529456 +0000 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Change: 2026-03-23 19:23:27.883529456 +0000 2026-03-23T19:30:13.263 INFO:teuthology.orchestra.run.vm04.stdout: Birth: 2026-03-23 19:23:27.883529456 +0000 2026-03-23T19:30:13.264 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-23T19:30:13.326 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records in 2026-03-23T19:30:13.327 INFO:teuthology.orchestra.run.vm04.stderr:1+0 records out 2026-03-23T19:30:13.327 INFO:teuthology.orchestra.run.vm04.stderr:512 bytes copied, 0.000141945 s, 3.6 MB/s 2026-03-23T19:30:13.328 DEBUG:teuthology.orchestra.run.vm04:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-23T19:30:13.384 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:13.384 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm04.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-23T19:30:13.384 INFO:tasks.ceph:Generating config... 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client] debug rbd = 20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client] log to stderr = False 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client] rbd default features = 125 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.385 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.385 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-03-23T19:30:13.385 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-03-23T19:30:13.385 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-03-23T19:30:13.385 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-23T19:30:13.385 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-23T19:30:13.385 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] bdev async discard = True 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] osd memory target = 1610612736 2026-03-23T19:30:13.385 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-03-23T19:30:13.385 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-03-23T19:30:13.386 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-03-23T19:30:13.460 INFO:teuthology.orchestra.run.vm04.stdout:creating /etc/ceph/cluster1.keyring 2026-03-23T19:30:13.462 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-03-23T19:30:13.540 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-23T19:30:13.605 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '192.168.123.104')] 2026-03-23T19:30:13.606 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.104', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-03-23T19:30:13.606 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:13.606 DEBUG:teuthology.orchestra.run.vm04:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-23T19:30:13.663 DEBUG:teuthology.orchestra.run.vm04:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.104 --print /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:monmaptool: generated fsid 2127a38f-edd0-4127-84bd-b60d939c2ed6 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:setting min_mon_release = tentacle 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:epoch 0 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:fsid 2127a38f-edd0-4127-84bd-b60d939c2ed6 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:last_changed 2026-03-23T19:30:13.738000+0000 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:created 2026-03-23T19:30:13.738000+0000 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:min_mon_release 20 (tentacle) 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:election_strategy: 1 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:0: [v2:192.168.123.104:3300/0,v1:192.168.123.104:6789/0] mon.a 2026-03-23T19:30:13.739 INFO:teuthology.orchestra.run.vm04.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-03-23T19:30:13.741 DEBUG:teuthology.orchestra.run.vm04:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-23T19:30:13.796 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 2127a38f-edd0-4127-84bd-b60d939c2ed6... 2026-03-23T19:30:13.797 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-03-23T19:30:13.838 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout:[global] 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: chdir = "" 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: auth supported = cephx 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: filestore xattr use omap = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon clock drift allowed = 1.000 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: osd crush chooseleaf type = 0 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: auth debug = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: ms die on old message = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: ms die on bug = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon pg warn max object skew = 0 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: # disable pg_autoscaler by default for new pools 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: osd pool default size = 2 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon osd allow primary affinity = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon osd allow pg remap = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on legacy crush tunables = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on crush straw calc version zero = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on no sortbitwise = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on osd down out interval zero = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on too few osds = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon_allow_pool_size_one = true 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: osd default data pool replay window = 5 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.879 INFO:teuthology.orchestra.run.vm04.stdout: mon allow pool delete = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon cluster log file level = debug 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug asserts on shutdown = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon health detail to clog = false 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon host = 192.168.123.104 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon client directed command retry = 5 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on pool no app = False 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: ms inject socket failures = 5000 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: fsid = 2127a38f-edd0-4127-84bd-b60d939c2ed6 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout:[osd] 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd journal size = 100 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd scrub load threshold = 5.0 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd scrub max interval = 600 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock profile = high_recovery_ops 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock skip benchmark = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd recover clone overlap = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd recovery max chunk = 1048576 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug shutdown = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug op order = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug verify stray on activate = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug trim objects = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd open classes on start = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug pg log writeout = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd deep scrub update digest min age = 30 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd map max advance = 10 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: journal zero on create = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: filestore ondisk finisher threads = 3 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: filestore apply finisher threads = 3 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: bdev debug aio = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd debug misdirected ops = true 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: bdev async discard = True 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: bdev enable discard = True 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: bluestore block size = 96636764160 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: bluestore fsck on mount = True 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug bluefs = 1/20 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug bluestore = 1/20 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug osd = 20 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: debug rocksdb = 4/10 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon osd full ratio = 0.9 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: mon osd nearfull ratio = 0.8 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd failsafe full ratio = 0.95 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:13.880 INFO:teuthology.orchestra.run.vm04.stdout: osd memory target = 1610612736 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: osd objectstore = bluestore 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[mgr] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug mgr = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug mon = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug auth = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mgr/telemetry/nag = false 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[mon] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug mon = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug paxos = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug auth = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon data avail warn = 5 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon mgr mkfs grace = 240 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon osd reporter subtree level = osd 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon osd prime pg temp = true 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: auth mon ticket ttl = 660 # 11m 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: auth service ticket ttl = 240 # 4m 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: # don't complain about insecure global_id in the test suite 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: # 1m isn't quite enough 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon_down_mkfs_grace = 2m 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_filestore_osds = false 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: rgw cache enabled = true 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: rgw enable ops log = true 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: rgw enable usage log = true 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug rbd = 20 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: debug rbd_mirror = 15 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: log to stderr = False 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: rbd default features = 125 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[mon.a] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.0] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.1] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.2] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.3] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.4] 2026-03-23T19:30:13.881 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.5] 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.6] 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.882 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout:[global] 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: chdir = "" 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: auth supported = cephx 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: filestore xattr use omap = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon clock drift allowed = 1.000 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: osd crush chooseleaf type = 0 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: auth debug = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: ms die on old message = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: ms die on bug = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon pg warn max object skew = 0 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: # disable pg_autoscaler by default for new pools 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default size = 2 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow primary affinity = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow pg remap = true 2026-03-23T19:30:13.885 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on legacy crush tunables = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on crush straw calc version zero = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on no sortbitwise = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on osd down out interval zero = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on too few osds = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon_allow_pool_size_one = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd default data pool replay window = 5 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon allow pool delete = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon cluster log file level = debug 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: debug asserts on shutdown = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon health detail to clog = false 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon host = 192.168.123.104 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon client directed command retry = 5 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on pool no app = False 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: ms inject socket failures = 5000 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: fsid = 2127a38f-edd0-4127-84bd-b60d939c2ed6 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout:[osd] 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd journal size = 100 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub load threshold = 5.0 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub max interval = 600 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock profile = high_recovery_ops 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock skip benchmark = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd recover clone overlap = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd recovery max chunk = 1048576 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd debug shutdown = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd debug op order = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd debug verify stray on activate = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd debug trim objects = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd open classes on start = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd debug pg log writeout = true 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.886 INFO:teuthology.orchestra.run.vm08.stdout: osd deep scrub update digest min age = 30 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd map max advance = 10 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: journal zero on create = true 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: filestore ondisk finisher threads = 3 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: filestore apply finisher threads = 3 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: bdev debug aio = true 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd debug misdirected ops = true 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: bdev async discard = True 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: bdev enable discard = True 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: bluestore block size = 96636764160 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: bluestore fsck on mount = True 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug bluefs = 1/20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug bluestore = 1/20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug osd = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug rocksdb = 4/10 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon osd full ratio = 0.9 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon osd nearfull ratio = 0.8 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd failsafe full ratio = 0.95 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd memory target = 1610612736 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: osd objectstore = bluestore 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout:[mgr] 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug mgr = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mgr/telemetry/nag = false 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout:[mon] 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug paxos = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon data avail warn = 5 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon mgr mkfs grace = 240 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon osd reporter subtree level = osd 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon osd prime pg temp = true 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: auth mon ticket ttl = 660 # 11m 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: auth service ticket ttl = 240 # 4m 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.887 INFO:teuthology.orchestra.run.vm08.stdout: # don't complain about insecure global_id in the test suite 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: # 1m isn't quite enough 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: mon_down_mkfs_grace = 2m 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_filestore_osds = false 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: rgw cache enabled = true 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable ops log = true 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable usage log = true 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: debug rbd = 20 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: debug rbd_mirror = 15 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: log to stderr = False 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: rbd default features = 125 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[mon.a] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.0] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.1] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.2] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.3] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.4] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.5] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.6] 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:13.888 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:13.896 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-03-23T19:30:13.896 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-03-23T19:30:13.974 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-23T19:30:13.975 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:13.975 DEBUG:teuthology.orchestra.run.vm04:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-03-23T19:30:13.990 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:13.990 DEBUG:teuthology.orchestra.run.vm04:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-03-23T19:30:14.044 INFO:tasks.ceph:Sending monmap to node ubuntu@vm04.local 2026-03-23T19:30:14.044 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:14.044 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/etc/ceph/cluster1.keyring 2026-03-23T19:30:14.044 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-23T19:30:14.120 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:14.120 DEBUG:teuthology.orchestra.run.vm04:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:14.177 INFO:tasks.ceph:Sending monmap to node ubuntu@vm08.local 2026-03-23T19:30:14.177 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:14.177 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/ceph/cluster1.keyring 2026-03-23T19:30:14.177 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-03-23T19:30:14.212 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:14.213 DEBUG:teuthology.orchestra.run.vm08:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:14.267 INFO:tasks.ceph:Setting up mon nodes... 2026-03-23T19:30:14.267 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-23T19:30:14.267 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-03-23T19:30:14.316 INFO:teuthology.orchestra.run.vm04.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-03-23T19:30:14.319 INFO:tasks.ceph:Setting up mds nodes... 2026-03-23T19:30:14.319 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-23T19:30:14.320 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-03-23T19:30:14.359 INFO:teuthology.orchestra.run.vm04.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-03-23T19:30:14.372 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-03-23T19:30:14.415 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-03-23T19:30:14.429 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-03-23T19:30:14.512 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-03-23T19:30:14.525 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-03-23T19:30:14.565 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-03-23T19:30:14.578 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-03-23T19:30:14.659 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-03-23T19:30:14.672 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-03-23T19:30:14.750 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-03-23T19:30:14.764 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-03-23T19:30:14.844 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-03-23T19:30:14.857 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-03-23T19:30:14.937 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-03-23T19:30:14.949 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-03-23T19:30:15.032 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-03-23T19:30:15.046 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-23T19:30:15.046 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm04.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-23T19:30:15.046 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-03-23T19:30:15.074 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:15.074 INFO:tasks.ceph:role: cluster1.osd.0 2026-03-23T19:30:15.074 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm04.local 2026-03-23T19:30:15.074 DEBUG:teuthology.orchestra.run.vm04:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:15.142 INFO:teuthology.orchestra.run.vm04.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:15.147 INFO:teuthology.orchestra.run.vm04.stdout:Discarding blocks...Done. 2026-03-23T19:30:15.150 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm04.local -o noatime 2026-03-23T19:30:15.150 DEBUG:teuthology.orchestra.run.vm04:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-03-23T19:30:15.227 DEBUG:teuthology.orchestra.run.vm04:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-03-23T19:30:15.299 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-03-23T19:30:15.368 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:15.369 INFO:tasks.ceph:role: cluster1.osd.1 2026-03-23T19:30:15.369 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm04.local 2026-03-23T19:30:15.369 DEBUG:teuthology.orchestra.run.vm04:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:15.436 INFO:teuthology.orchestra.run.vm04.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:15.442 INFO:teuthology.orchestra.run.vm04.stdout:Discarding blocks...Done. 2026-03-23T19:30:15.444 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm04.local -o noatime 2026-03-23T19:30:15.444 DEBUG:teuthology.orchestra.run.vm04:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-03-23T19:30:15.518 DEBUG:teuthology.orchestra.run.vm04:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-03-23T19:30:15.588 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-03-23T19:30:15.658 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:15.659 INFO:tasks.ceph:role: cluster1.osd.2 2026-03-23T19:30:15.659 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm04.local 2026-03-23T19:30:15.659 DEBUG:teuthology.orchestra.run.vm04:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-23T19:30:15.726 INFO:teuthology.orchestra.run.vm04.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:15.726 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:15.726 INFO:teuthology.orchestra.run.vm04.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:15.726 INFO:teuthology.orchestra.run.vm04.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:15.727 INFO:teuthology.orchestra.run.vm04.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:15.731 INFO:teuthology.orchestra.run.vm04.stdout:Discarding blocks...Done. 2026-03-23T19:30:15.734 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm04.local -o noatime 2026-03-23T19:30:15.734 DEBUG:teuthology.orchestra.run.vm04:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-03-23T19:30:15.805 DEBUG:teuthology.orchestra.run.vm04:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-03-23T19:30:15.878 DEBUG:teuthology.orchestra.run.vm04:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:15.964 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:15.961+0000 7f77cf012900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-03-23T19:30:15.965 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:15.962+0000 7f77cf012900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-03-23T19:30:15.965 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:15.962+0000 7f77cf012900 -1 bdev(0x563f11bbf800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:15.965 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:15.962+0000 7f77cf012900 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-03-23T19:30:16.578 DEBUG:teuthology.orchestra.run.vm04:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-03-23T19:30:16.609 DEBUG:teuthology.orchestra.run.vm04:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:16.691 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:16.688+0000 7fa0f891a900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-03-23T19:30:16.692 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:16.689+0000 7fa0f891a900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-03-23T19:30:16.692 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:16.689+0000 7fa0f891a900 -1 bdev(0x55a3b8ddf800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:16.692 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:16.689+0000 7fa0f891a900 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-03-23T19:30:17.352 DEBUG:teuthology.orchestra.run.vm04:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-03-23T19:30:17.418 DEBUG:teuthology.orchestra.run.vm04:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:17.496 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:17.494+0000 7fa5098d9900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-03-23T19:30:17.497 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:17.494+0000 7fa5098d9900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-03-23T19:30:17.497 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:17.494+0000 7fa5098d9900 -1 bdev(0x55c67c411800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:17.497 INFO:teuthology.orchestra.run.vm04.stderr:2026-03-23T19:30:17.494+0000 7fa5098d9900 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-03-23T19:30:18.132 DEBUG:teuthology.orchestra.run.vm04:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-03-23T19:30:18.198 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-23T19:30:18.198 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:18.198 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-03-23T19:30:18.261 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:18.261 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-03-23T19:30:18.328 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:18.328 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-03-23T19:30:18.395 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:18.395 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-03-23T19:30:18.463 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:18.463 DEBUG:teuthology.orchestra.run.vm04:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-03-23T19:30:18.523 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.523 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-03-23T19:30:18.539 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.539 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-03-23T19:30:18.593 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.593 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-03-23T19:30:18.646 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.647 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-03-23T19:30:18.702 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.702 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-03-23T19:30:18.759 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.759 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-03-23T19:30:18.817 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.817 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-03-23T19:30:18.875 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:18.875 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-03-23T19:30:18.931 INFO:tasks.ceph:Adding keys to all mons... 2026-03-23T19:30:18.931 DEBUG:teuthology.orchestra.run.vm04:> sudo tee -a /etc/ceph/cluster1.keyring 2026-03-23T19:30:18.953 INFO:teuthology.orchestra.run.vm04.stdout:[mgr.x] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpObvKEhAArRWWMqPHtrsz8MAR8fDY9Q== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[osd.0] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDHlMFpAoJnORAAWlA5XA0gjYG3zky9hDDAmw== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[osd.1] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDIlMFp8pMhKRAAw5PahdRkLTdl3u/nWC9M1w== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[osd.2] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDJlMFp206DHRAAXjvMwnWyAr5/aCnGEHOJIQ== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.0] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpew9UFRAA+MojEXDzesGCRwRIpd2OeQ== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpZW+sGBAA5mcL98kqSmiBPO9Bu3Yr0A== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.0] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpNuJxHhAAzcH0xucKl4YuR7btCgw5qQ== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.1] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFptMmbIRAAks9OeSQSbJxNIbGhvQWUuw== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.2] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpckg0JxAA3UGNqXgqx5mg5l2eI7b9HA== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.3] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFp19WpLBAA6cw8eAcg6P810Dl1fNqJLQ== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.4] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpUas4MhAApYakVjh/q43RE8ANhtlJDg== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.5] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDGlMFpiX7GNxAAsNnkjn4sctK9b4jXGrHCoA== 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.6] 2026-03-23T19:30:18.954 INFO:teuthology.orchestra.run.vm04.stdout: key = AQDHlMFpDpLdARAAiXITMsa+pN+tkaPDCCiEmQ== 2026-03-23T19:30:18.954 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-23T19:30:19.036 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:19.123 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:19.167 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:19.211 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.262 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.313 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.359 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.407 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.455 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.501 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.546 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.592 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:19.637 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-23T19:30:19.637 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-03-23T19:30:19.662 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-03-23T19:30:19.759 DEBUG:teuthology.orchestra.run.vm04:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-03-23T19:30:19.786 DEBUG:teuthology.orchestra.run.vm04:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-03-23T19:30:19.843 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-03-23T19:30:19.843 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-03-23T19:30:19.843 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-03-23T19:30:19.886 INFO:tasks.ceph.cluster1.mon.a:Started 2026-03-23T19:30:19.886 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-03-23T19:30:19.886 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-03-23T19:30:19.886 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-03-23T19:30:19.888 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-03-23T19:30:19.888 DEBUG:tasks.ceph:set 0 configs 2026-03-23T19:30:19.888 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 config dump 2026-03-23T19:30:20.010 INFO:teuthology.orchestra.run.vm04.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-23T19:30:20.020 INFO:tasks.ceph:Setting crush tunables to default 2026-03-23T19:30:20.021 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 osd crush tunables default 2026-03-23T19:30:20.139 INFO:teuthology.orchestra.run.vm04.stderr:adjusted tunables profile to default 2026-03-23T19:30:20.151 INFO:tasks.ceph:check_enable_crimson: False 2026-03-23T19:30:20.151 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-03-23T19:30:20.151 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:20.151 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-03-23T19:30:20.179 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:20.179 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-03-23T19:30:20.244 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:20.244 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-03-23T19:30:20.308 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 osd new 4ddb3b53-149c-4477-8cf0-20b408f9c667 0 2026-03-23T19:30:20.471 INFO:teuthology.orchestra.run.vm04.stdout:0 2026-03-23T19:30:20.481 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 osd new 905f6bd4-a644-41c0-8cb6-d3acdd71708a 1 2026-03-23T19:30:20.600 INFO:teuthology.orchestra.run.vm04.stdout:1 2026-03-23T19:30:20.610 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 osd new 630c2e98-f3f7-4b17-824c-cc35ddbeb468 2 2026-03-23T19:30:20.727 INFO:teuthology.orchestra.run.vm04.stdout:2 2026-03-23T19:30:20.738 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-03-23T19:30:20.738 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-03-23T19:30:20.739 INFO:tasks.ceph.cluster1.osd.0:Started 2026-03-23T19:30:20.739 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-03-23T19:30:20.739 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-03-23T19:30:20.740 INFO:tasks.ceph.cluster1.osd.1:Started 2026-03-23T19:30:20.740 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-03-23T19:30:20.740 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-03-23T19:30:20.743 INFO:tasks.ceph.cluster1.osd.2:Started 2026-03-23T19:30:20.743 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:30:20.892 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:20.892 INFO:teuthology.orchestra.run.vm04.stdout:{"epoch":5,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:30:20.724262+0000","last_up_change":"0.000000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:20.901 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-03-23T19:30:20.901 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-23T19:30:20.917 INFO:tasks.ceph.cluster1.osd.1.vm04.stderr:2026-03-23T19:30:20.914+0000 7f9437225900 -1 Falling back to public interface 2026-03-23T19:30:20.929 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:30:20.926+0000 7f9871425900 -1 Falling back to public interface 2026-03-23T19:30:20.939 INFO:tasks.ceph.cluster1.osd.0.vm04.stderr:2026-03-23T19:30:20.936+0000 7f04578a6900 -1 Falling back to public interface 2026-03-23T19:30:20.979 INFO:tasks.ceph.cluster1.mgr.x.vm04.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-23T19:30:20.979 INFO:tasks.ceph.cluster1.mgr.x.vm04.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-23T19:30:20.979 INFO:tasks.ceph.cluster1.mgr.x.vm04.stderr: from numpy import show_config as show_numpy_config 2026-03-23T19:30:21.203 DEBUG:teuthology.orchestra.run.vm04:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:30:21.229 INFO:tasks.ceph.cluster1.osd.1.vm04.stderr:2026-03-23T19:30:21.227+0000 7f9437225900 -1 osd.1 0 log_to_monitors true 2026-03-23T19:30:21.257 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:30:21.254+0000 7f9871425900 -1 osd.2 0 log_to_monitors true 2026-03-23T19:30:21.294 INFO:tasks.ceph.cluster1.osd.0.vm04.stderr:2026-03-23T19:30:21.291+0000 7f04578a6900 -1 osd.0 0 log_to_monitors true 2026-03-23T19:30:21.325 INFO:teuthology.misc.health.vm04.stdout: 2026-03-23T19:30:21.325 INFO:teuthology.misc.health.vm04.stdout:{"epoch":5,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:30:20.724262+0000","last_up_change":"0.000000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:21.332 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-03-23T19:30:22.996 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:30:22.993+0000 7f986cb93640 -1 osd.2 0 waiting for initial osdmap 2026-03-23T19:30:23.000 INFO:tasks.ceph.cluster1.osd.1.vm04.stderr:2026-03-23T19:30:22.997+0000 7f9432993640 -1 osd.1 0 waiting for initial osdmap 2026-03-23T19:30:23.000 INFO:tasks.ceph.cluster1.osd.0.vm04.stderr:2026-03-23T19:30:22.997+0000 7f0454047640 -1 osd.0 0 waiting for initial osdmap 2026-03-23T19:30:23.010 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:30:23.008+0000 7f98681aa640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:23.010 INFO:tasks.ceph.cluster1.osd.1.vm04.stderr:2026-03-23T19:30:23.008+0000 7f942dfaa640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:23.015 INFO:tasks.ceph.cluster1.osd.0.vm04.stderr:2026-03-23T19:30:23.012+0000 7f044e629640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:23.536 INFO:tasks.ceph.cluster1.mgr.x.vm04.stderr:2026-03-23T19:30:23.533+0000 7f373f915640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-23T19:30:27.634 DEBUG:teuthology.orchestra.run.vm04:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:30:27.831 INFO:teuthology.misc.health.vm04.stdout: 2026-03-23T19:30:27.831 INFO:teuthology.misc.health.vm04.stdout:{"epoch":11,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:30:27.011454+0000","last_up_change":"2026-03-23T19:30:23.994669+0000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:24.542060+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6816","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6817","nonce":1787023239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6818","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6819","nonce":1787023239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6822","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6823","nonce":1787023239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6820","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6821","nonce":1787023239}]},"public_addr":"192.168.123.104:6817/1787023239","cluster_addr":"192.168.123.104:6819/1787023239","heartbeat_back_addr":"192.168.123.104:6823/1787023239","heartbeat_front_addr":"192.168.123.104:6821/1787023239","state":["exists","up"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6800","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6801","nonce":3959752833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6802","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6803","nonce":3959752833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6806","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6807","nonce":3959752833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6804","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6805","nonce":3959752833}]},"public_addr":"192.168.123.104:6801/3959752833","cluster_addr":"192.168.123.104:6803/3959752833","heartbeat_back_addr":"192.168.123.104:6807/3959752833","heartbeat_front_addr":"192.168.123.104:6805/3959752833","state":["exists","up"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6808","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6809","nonce":1186798788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6810","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6811","nonce":1186798788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6814","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6815","nonce":1186798788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6812","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6813","nonce":1186798788}]},"public_addr":"192.168.123.104:6809/1186798788","cluster_addr":"192.168.123.104:6811/1186798788","heartbeat_back_addr":"192.168.123.104:6815/1186798788","heartbeat_front_addr":"192.168.123.104:6813/1186798788","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.258841+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.196343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.257168+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:27.841 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-03-23T19:30:27.841 INFO:tasks.ceph:Creating RBD pool 2026-03-23T19:30:27.841 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-03-23T19:30:29.022 INFO:teuthology.orchestra.run.vm04.stderr:pool 'rbd' created 2026-03-23T19:30:29.037 DEBUG:teuthology.orchestra.run.vm04:> rbd --cluster cluster1 pool init rbd 2026-03-23T19:30:32.044 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-03-23T19:30:32.044 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-03-23T19:30:32.044 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-23T19:30:32.246 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:32.258 INFO:teuthology.orchestra.run.vm04.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-23T19:30:32.258 INFO:tasks.ceph_manager:config epoch is 1 2026-03-23T19:30:32.258 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-03-23T19:30:32.258 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-03-23T19:30:32.258 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-03-23T19:30:32.507 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:32.518 INFO:teuthology.orchestra.run.vm04.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6824","nonce":1129433442},{"type":"v1","addr":"192.168.123.104:6825","nonce":1129433442}]},"active_addr":"192.168.123.104:6825/1129433442","active_change":"2026-03-23T19:30:22.522444+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":194462287}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":2957640104}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":4162031938}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":2059710460}]}]} 2026-03-23T19:30:32.519 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-03-23T19:30:32.519 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-03-23T19:30:32.519 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:30:32.720 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:32.720 INFO:teuthology.orchestra.run.vm04.stdout:{"epoch":15,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:30:32.028481+0000","last_up_change":"2026-03-23T19:30:23.994669+0000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:24.542060+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:28.044741+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6816","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6817","nonce":1787023239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6818","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6819","nonce":1787023239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6822","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6823","nonce":1787023239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6820","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6821","nonce":1787023239}]},"public_addr":"192.168.123.104:6817/1787023239","cluster_addr":"192.168.123.104:6819/1787023239","heartbeat_back_addr":"192.168.123.104:6823/1787023239","heartbeat_front_addr":"192.168.123.104:6821/1787023239","state":["exists","up"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6800","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6801","nonce":3959752833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6802","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6803","nonce":3959752833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6806","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6807","nonce":3959752833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6804","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6805","nonce":3959752833}]},"public_addr":"192.168.123.104:6801/3959752833","cluster_addr":"192.168.123.104:6803/3959752833","heartbeat_back_addr":"192.168.123.104:6807/3959752833","heartbeat_front_addr":"192.168.123.104:6805/3959752833","state":["exists","up"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6808","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6809","nonce":1186798788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6810","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6811","nonce":1186798788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6814","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6815","nonce":1186798788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6812","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6813","nonce":1186798788}]},"public_addr":"192.168.123.104:6809/1186798788","cluster_addr":"192.168.123.104:6811/1186798788","heartbeat_back_addr":"192.168.123.104:6815/1186798788","heartbeat_front_addr":"192.168.123.104:6813/1186798788","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.258841+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.196343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.257168+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:32.732 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-03-23T19:30:32.732 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:30:32.926 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:32.926 INFO:teuthology.orchestra.run.vm04.stdout:{"epoch":15,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:30:32.028481+0000","last_up_change":"2026-03-23T19:30:23.994669+0000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:24.542060+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:28.044741+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6816","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6817","nonce":1787023239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6818","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6819","nonce":1787023239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6822","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6823","nonce":1787023239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6820","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6821","nonce":1787023239}]},"public_addr":"192.168.123.104:6817/1787023239","cluster_addr":"192.168.123.104:6819/1787023239","heartbeat_back_addr":"192.168.123.104:6823/1787023239","heartbeat_front_addr":"192.168.123.104:6821/1787023239","state":["exists","up"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6800","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6801","nonce":3959752833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6802","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6803","nonce":3959752833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6806","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6807","nonce":3959752833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6804","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6805","nonce":3959752833}]},"public_addr":"192.168.123.104:6801/3959752833","cluster_addr":"192.168.123.104:6803/3959752833","heartbeat_back_addr":"192.168.123.104:6807/3959752833","heartbeat_front_addr":"192.168.123.104:6805/3959752833","state":["exists","up"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6808","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6809","nonce":1186798788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6810","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6811","nonce":1186798788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6814","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6815","nonce":1186798788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6812","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6813","nonce":1186798788}]},"public_addr":"192.168.123.104:6809/1186798788","cluster_addr":"192.168.123.104:6811/1186798788","heartbeat_back_addr":"192.168.123.104:6815/1186798788","heartbeat_front_addr":"192.168.123.104:6813/1186798788","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.258841+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.196343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.257168+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:32.938 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-03-23T19:30:32.938 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-03-23T19:30:32.938 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-03-23T19:30:33.062 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:33.062 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-23T19:30:33.066 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:33.067 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:33.067 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-23T19:30:33.067 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-23T19:30:33.287 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:33.306 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-03-23T19:30:33.330 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:33.337 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:33.344 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-03-23T19:30:33.351 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-03-23T19:30:34.307 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-23T19:30:34.346 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-23T19:30:34.351 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-23T19:30:34.516 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:34.532 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-03-23T19:30:34.559 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:34.574 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-03-23T19:30:34.620 INFO:teuthology.orchestra.run.vm04.stdout:34359738370 2026-03-23T19:30:34.635 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-03-23T19:30:35.540 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-03-23T19:30:35.575 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-03-23T19:30:35.636 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-03-23T19:30:35.768 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:35.784 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-03-23T19:30:35.784 DEBUG:teuthology.parallel:result is None 2026-03-23T19:30:35.816 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:35.829 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-03-23T19:30:35.829 DEBUG:teuthology.parallel:result is None 2026-03-23T19:30:35.852 INFO:teuthology.orchestra.run.vm04.stdout:34359738371 2026-03-23T19:30:35.865 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.2 2026-03-23T19:30:35.865 DEBUG:teuthology.parallel:result is None 2026-03-23T19:30:35.865 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-03-23T19:30:35.865 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:30:36.120 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:36.121 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:30:36.133 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-03-23T19:30:34.530335+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82096,"kb_used_data":1552,"kb_used_omap":25,"kb_used_meta":80422,"kb_avail":283033424,"statfs":{"total":289910292480,"available":289826226176,"internally_reserved":0,"allocated":1589248,"data_stored":1295581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25681,"internal_metadata":82353071},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.507533"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032402+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:30:32.032402+0000","last_peered":"2026-03-23T19:30:32.032402+0000","last_clean":"2026-03-23T19:30:32.032402+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:30:32.032402+0000","last_undegraded":"2026-03-23T19:30:32.032402+0000","last_fullsized":"2026-03-23T19:30:32.032402+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032804+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:30:32.032804+0000","last_peered":"2026-03-23T19:30:32.032804+0000","last_clean":"2026-03-23T19:30:32.032804+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:30:32.032804+0000","last_undegraded":"2026-03-23T19:30:32.032804+0000","last_fullsized":"2026-03-23T19:30:32.032804+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032385+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:30:32.032385+0000","last_peered":"2026-03-23T19:30:32.032385+0000","last_clean":"2026-03-23T19:30:32.032385+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:30:32.032385+0000","last_undegraded":"2026-03-23T19:30:32.032385+0000","last_fullsized":"2026-03-23T19:30:32.032385+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032947+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:30:32.032947+0000","last_peered":"2026-03-23T19:30:32.032947+0000","last_clean":"2026-03-23T19:30:32.032947+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:30:32.032947+0000","last_undegraded":"2026-03-23T19:30:32.032947+0000","last_fullsized":"2026-03-23T19:30:32.032947+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.037291+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:30:32.037291+0000","last_peered":"2026-03-23T19:30:32.037291+0000","last_clean":"2026-03-23T19:30:32.037291+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:30:32.037291+0000","last_undegraded":"2026-03-23T19:30:32.037291+0000","last_fullsized":"2026-03-23T19:30:32.037291+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.203404+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:30:32.203404+0000","last_peered":"2026-03-23T19:30:32.203404+0000","last_clean":"2026-03-23T19:30:32.203404+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:30:32.203404+0000","last_undegraded":"2026-03-23T19:30:32.203404+0000","last_fullsized":"2026-03-23T19:30:32.203404+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.203472+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:30:32.203472+0000","last_peered":"2026-03-23T19:30:32.203472+0000","last_clean":"2026-03-23T19:30:32.203472+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:30:32.203472+0000","last_undegraded":"2026-03-23T19:30:32.203472+0000","last_fullsized":"2026-03-23T19:30:32.203472+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032687+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:30:32.032687+0000","last_peered":"2026-03-23T19:30:32.032687+0000","last_clean":"2026-03-23T19:30:32.032687+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:30:32.032687+0000","last_undegraded":"2026-03-23T19:30:32.032687+0000","last_fullsized":"2026-03-23T19:30:32.032687+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'92","reported_seq":134,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032304+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:30:32.032304+0000","last_peered":"2026-03-23T19:30:32.032304+0000","last_clean":"2026-03-23T19:30:32.032304+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:30:32.032304+0000","last_undegraded":"2026-03-23T19:30:32.032304+0000","last_fullsized":"2026-03-23T19:30:32.032304+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":131072,"data_stored":38269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:30:36.133 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:30:36.335 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:36.335 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:30:36.348 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-03-23T19:30:34.530335+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82096,"kb_used_data":1552,"kb_used_omap":25,"kb_used_meta":80422,"kb_avail":283033424,"statfs":{"total":289910292480,"available":289826226176,"internally_reserved":0,"allocated":1589248,"data_stored":1295581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25681,"internal_metadata":82353071},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.507533"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032402+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:30:32.032402+0000","last_peered":"2026-03-23T19:30:32.032402+0000","last_clean":"2026-03-23T19:30:32.032402+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:30:32.032402+0000","last_undegraded":"2026-03-23T19:30:32.032402+0000","last_fullsized":"2026-03-23T19:30:32.032402+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032804+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:30:32.032804+0000","last_peered":"2026-03-23T19:30:32.032804+0000","last_clean":"2026-03-23T19:30:32.032804+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:30:32.032804+0000","last_undegraded":"2026-03-23T19:30:32.032804+0000","last_fullsized":"2026-03-23T19:30:32.032804+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032385+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:30:32.032385+0000","last_peered":"2026-03-23T19:30:32.032385+0000","last_clean":"2026-03-23T19:30:32.032385+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:30:32.032385+0000","last_undegraded":"2026-03-23T19:30:32.032385+0000","last_fullsized":"2026-03-23T19:30:32.032385+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032947+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:30:32.032947+0000","last_peered":"2026-03-23T19:30:32.032947+0000","last_clean":"2026-03-23T19:30:32.032947+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:30:32.032947+0000","last_undegraded":"2026-03-23T19:30:32.032947+0000","last_fullsized":"2026-03-23T19:30:32.032947+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.037291+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:30:32.037291+0000","last_peered":"2026-03-23T19:30:32.037291+0000","last_clean":"2026-03-23T19:30:32.037291+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:30:32.037291+0000","last_undegraded":"2026-03-23T19:30:32.037291+0000","last_fullsized":"2026-03-23T19:30:32.037291+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.203404+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:30:32.203404+0000","last_peered":"2026-03-23T19:30:32.203404+0000","last_clean":"2026-03-23T19:30:32.203404+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:30:32.203404+0000","last_undegraded":"2026-03-23T19:30:32.203404+0000","last_fullsized":"2026-03-23T19:30:32.203404+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.203472+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:30:32.203472+0000","last_peered":"2026-03-23T19:30:32.203472+0000","last_clean":"2026-03-23T19:30:32.203472+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:30:32.203472+0000","last_undegraded":"2026-03-23T19:30:32.203472+0000","last_fullsized":"2026-03-23T19:30:32.203472+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032687+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:30:32.032687+0000","last_peered":"2026-03-23T19:30:32.032687+0000","last_clean":"2026-03-23T19:30:32.032687+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:30:32.032687+0000","last_undegraded":"2026-03-23T19:30:32.032687+0000","last_fullsized":"2026-03-23T19:30:32.032687+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'92","reported_seq":134,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:32.032304+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:30:32.032304+0000","last_peered":"2026-03-23T19:30:32.032304+0000","last_clean":"2026-03-23T19:30:32.032304+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:30:32.032304+0000","last_undegraded":"2026-03-23T19:30:32.032304+0000","last_fullsized":"2026-03-23T19:30:32.032304+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":131072,"data_stored":38269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:30:36.349 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-03-23T19:30:36.349 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-03-23T19:30:36.349 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-03-23T19:30:36.349 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-03-23T19:30:36.584 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:36.584 INFO:teuthology.orchestra.run.vm04.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-23T19:30:36.596 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-03-23T19:30:36.596 INFO:teuthology.run_tasks:Running task ceph... 2026-03-23T19:30:36.598 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-03-23T19:30:36.598 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-03-23T19:30:36.598 INFO:tasks.ceph:ctx.config {'archive_path': '/archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508', 'branch': 'tentacle', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock}', 'email': None, 'first_in_suite': False, 'flavor': 'default', 'job_id': '3508', 'last_in_suite': False, 'machine_type': 'vps', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'tentacle'}, 'ansible.cephlab': {'branch': 'main', 'repo': 'https://github.com/kshtsk/ceph-cm-ansible.git', 'skip_tags': 'nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs', 'vars': {'logical_volumes': {'lv_1': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_2': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_3': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}, 'lv_4': {'scratch_dev': True, 'size': '25%VG', 'vg': 'vg_nvme'}}, 'timezone': 'UTC', 'volume_groups': {'vg_nvme': {'pvs': '/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde'}}}}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'cephadm': {'cephadm_binary_url': 'https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}, 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'tt-tentacle', 'sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4'}}, 'owner': 'kyr', 'priority': 1000, 'repo': 'https://github.com/ceph/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 3051, 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'sleep_before_teardown': 0, 'subset': '1/128', 'suite': 'rbd', 'suite_branch': 'tt-tentacle', 'suite_path': '/home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/kshtsk/ceph.git', 'suite_sha1': '0392f78529848ec72469e8e431875cb98d3a5fb4', 'targets': {'vm04.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhg/2GSak8G0a+2w5RjKmf+3iY5nURPEtfBQShXyIoJgJqjg2pYSFw3UJqPnH/xW3ngd6idIBZJS/903A7dyd0=', 'vm08.local': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYsXVZtZMfIMUf0VakwgvN+ByRF0wMwOF9RQ9ibQa2la0cEC5/9df3h1qwh+9fObltloTOt+jwhJr08L+Vb+D8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '70f8415b300f041766fa27faf7d5472699e32388', 'extra_system_packages': {'deb': ['python3-jmespath', 'python3-xmltodict', 's3cmd'], 'rpm': ['bzip2', 'perl-Test-Harness', 'python3-jmespath', 'python3-xmltodict', 's3cmd']}}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '70f8415b300f041766fa27faf7d5472699e32388'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'clyso-debian-13', 'teuthology_repo': 'https://github.com/clyso/teuthology', 'teuthology_sha1': '1c580df7a9c7c2aadc272da296344fd99f27c444', 'timestamp': '2026-03-20_22:04:26', 'tube': 'vps', 'user': 'kyr', 'verbose': False, 'worker_log': '/home/teuthos/.teuthology/dispatcher/dispatcher.vps.2366871'} 2026-03-23T19:30:36.598 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-03-23T19:30:36.600 DEBUG:teuthology.orchestra.run.vm08:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-03-23T19:30:36.615 DEBUG:teuthology.orchestra.run.vm04:> sudo install -d -m0777 -- /var/run/ceph 2026-03-23T19:30:36.656 DEBUG:teuthology.orchestra.run.vm08:> sudo install -d -m0777 -- /var/run/ceph 2026-03-23T19:30:36.681 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:36.681 DEBUG:teuthology.orchestra.run.vm08:> dd if=/scratch_devs of=/dev/stdout 2026-03-23T19:30:36.737 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-03-23T19:30:36.737 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_1 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 648 Links: 1 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-23 19:30:08.610959634 +0000 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-23 19:23:53.391104761 +0000 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-23 19:23:53.391104761 +0000 2026-03-23T19:30:36.795 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-23 19:23:53.391104761 +0000 2026-03-23T19:30:36.795 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-03-23T19:30:36.862 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-23T19:30:36.862 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-23T19:30:36.862 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000152135 s, 3.4 MB/s 2026-03-23T19:30:36.863 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-03-23T19:30:36.919 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_2 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 680 Links: 1 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-23 19:30:08.611959636 +0000 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-23 19:23:53.585105055 +0000 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-23 19:23:53.585105055 +0000 2026-03-23T19:30:36.974 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-23 19:23:53.585105055 +0000 2026-03-23T19:30:36.975 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-03-23T19:30:37.037 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-23T19:30:37.037 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-23T19:30:37.037 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000159148 s, 3.2 MB/s 2026-03-23T19:30:37.038 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-03-23T19:30:37.097 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_3 2026-03-23T19:30:37.155 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-03-23T19:30:37.155 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:37.155 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 729 Links: 1 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-23 19:30:08.611959636 +0000 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-23 19:23:53.785105357 +0000 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-23 19:23:53.785105357 +0000 2026-03-23T19:30:37.156 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-23 19:23:53.785105357 +0000 2026-03-23T19:30:37.156 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-03-23T19:30:37.219 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-23T19:30:37.219 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-23T19:30:37.219 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000152075 s, 3.4 MB/s 2026-03-23T19:30:37.220 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-03-23T19:30:37.277 DEBUG:teuthology.orchestra.run.vm08:> stat /dev/vg_nvme/lv_4 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Device: 6h/6d Inode: 769 Links: 1 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Context: system_u:object_r:device_t:s0 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Access: 2026-03-23 19:30:08.612959637 +0000 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Modify: 2026-03-23 19:23:53.982105655 +0000 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout:Change: 2026-03-23 19:23:53.982105655 +0000 2026-03-23T19:30:37.333 INFO:teuthology.orchestra.run.vm08.stdout: Birth: 2026-03-23 19:23:53.982105655 +0000 2026-03-23T19:30:37.333 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-03-23T19:30:37.397 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records in 2026-03-23T19:30:37.397 INFO:teuthology.orchestra.run.vm08.stderr:1+0 records out 2026-03-23T19:30:37.397 INFO:teuthology.orchestra.run.vm08.stderr:512 bytes copied, 0.000136966 s, 3.7 MB/s 2026-03-23T19:30:37.398 DEBUG:teuthology.orchestra.run.vm08:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-03-23T19:30:37.454 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:37.454 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@vm08.local'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-23T19:30:37.454 INFO:tasks.ceph:Generating config... 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client] debug rbd = 20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client] log to stderr = False 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client] rbd default features = 125 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.455 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.455 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-03-23T19:30:37.455 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-03-23T19:30:37.455 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-03-23T19:30:37.455 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[mgr] debug ms = 1 2026-03-23T19:30:37.455 INFO:tasks.ceph:[mon] debug mon = 20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[mon] debug ms = 1 2026-03-23T19:30:37.455 INFO:tasks.ceph:[mon] debug paxos = 20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] bdev async discard = True 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] debug ms = 1 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] debug osd = 20 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:37.455 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-03-23T19:30:37.456 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-03-23T19:30:37.456 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-03-23T19:30:37.456 INFO:tasks.ceph:[osd] osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:37.456 INFO:tasks.ceph:[osd] osd memory target = 1610612736 2026-03-23T19:30:37.456 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-03-23T19:30:37.456 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-03-23T19:30:37.456 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-03-23T19:30:37.533 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.keyring 2026-03-23T19:30:37.536 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-03-23T19:30:37.613 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-23T19:30:37.679 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '192.168.123.108')] 2026-03-23T19:30:37.679 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '192.168.123.108', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd mclock iops capacity threshold hdd': 49000, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-03-23T19:30:37.679 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:37.679 DEBUG:teuthology.orchestra.run.vm08:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-03-23T19:30:37.735 DEBUG:teuthology.orchestra.run.vm08:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 192.168.123.108 --print /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:monmaptool: generated fsid 60776ecd-5e60-43ff-94d1-72ba986d0b0a 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:setting min_mon_release = tentacle 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:epoch 0 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:fsid 60776ecd-5e60-43ff-94d1-72ba986d0b0a 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:last_changed 2026-03-23T19:30:37.807694+0000 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:created 2026-03-23T19:30:37.807694+0000 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:min_mon_release 20 (tentacle) 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:election_strategy: 1 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:0: [v2:192.168.123.108:3300/0,v1:192.168.123.108:6789/0] mon.a 2026-03-23T19:30:37.809 INFO:teuthology.orchestra.run.vm08.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-03-23T19:30:37.810 DEBUG:teuthology.orchestra.run.vm08:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-03-23T19:30:37.866 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 60776ecd-5e60-43ff-94d1-72ba986d0b0a... 2026-03-23T19:30:37.866 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-03-23T19:30:37.868 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout:[global] 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: chdir = "" 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: auth supported = cephx 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: filestore xattr use omap = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon clock drift allowed = 1.000 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd crush chooseleaf type = 0 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: auth debug = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: ms die on old message = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: ms die on bug = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon pg warn max object skew = 0 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: # disable pg_autoscaler by default for new pools 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd pool default size = 2 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon osd allow primary affinity = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon osd allow pg remap = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on legacy crush tunables = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on crush straw calc version zero = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on no sortbitwise = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on osd down out interval zero = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on too few osds = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon_allow_pool_size_one = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd default data pool replay window = 5 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon allow pool delete = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon cluster log file level = debug 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: debug asserts on shutdown = true 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon health detail to clog = false 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon host = 192.168.123.108 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon client directed command retry = 5 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: mon warn on pool no app = False 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: ms inject socket failures = 5000 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: fsid = 60776ecd-5e60-43ff-94d1-72ba986d0b0a 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout:[osd] 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: osd journal size = 100 2026-03-23T19:30:37.909 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd scrub load threshold = 5.0 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd scrub max interval = 600 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock profile = high_recovery_ops 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock skip benchmark = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd recover clone overlap = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd recovery max chunk = 1048576 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug shutdown = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug op order = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug verify stray on activate = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug trim objects = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd open classes on start = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug pg log writeout = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd deep scrub update digest min age = 30 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd map max advance = 10 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: journal zero on create = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: filestore ondisk finisher threads = 3 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: filestore apply finisher threads = 3 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: bdev debug aio = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd debug misdirected ops = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: bdev async discard = True 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: bdev enable discard = True 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: bluestore block size = 96636764160 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: bluestore fsck on mount = True 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug bluefs = 1/20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug bluestore = 1/20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug osd = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug rocksdb = 4/10 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon osd full ratio = 0.9 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon osd nearfull ratio = 0.8 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd failsafe full ratio = 0.95 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd memory target = 1610612736 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: osd objectstore = bluestore 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout:[mgr] 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug mgr = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug mon = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug auth = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mgr/telemetry/nag = false 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout:[mon] 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug ms = 1 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug mon = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug paxos = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: debug auth = 20 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon data avail warn = 5 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon mgr mkfs grace = 240 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon osd reporter subtree level = osd 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon osd prime pg temp = true 2026-03-23T19:30:37.910 INFO:teuthology.orchestra.run.vm04.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: auth mon ticket ttl = 660 # 11m 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: auth service ticket ttl = 240 # 4m 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: # don't complain about insecure global_id in the test suite 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: # 1m isn't quite enough 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: mon_down_mkfs_grace = 2m 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: mon_warn_on_filestore_osds = false 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: rgw cache enabled = true 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: rgw enable ops log = true 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: rgw enable usage log = true 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: debug rbd = 20 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: debug rbd_mirror = 15 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: log to stderr = False 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: rbd default features = 125 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[mon.a] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.0] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.1] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.2] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.3] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.4] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.5] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout:[client.mirror.6] 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.911 INFO:teuthology.orchestra.run.vm04.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout:[global] 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: chdir = "" 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: auth supported = cephx 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: filestore xattr use omap = true 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: mon clock drift allowed = 1.000 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: osd crush chooseleaf type = 0 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: auth debug = true 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: ms die on old message = true 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: ms die on bug = true 2026-03-23T19:30:37.952 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon max pg per osd = 10000 # >= luminous 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon pg warn max object skew = 0 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: # disable pg_autoscaler by default for new pools 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd_pool_default_pg_autoscale_mode = off 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default size = 2 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow primary affinity = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon osd allow pg remap = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on legacy crush tunables = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on crush straw calc version zero = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on no sortbitwise = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on osd down out interval zero = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on too few osds = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_pool_no_redundancy = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon_allow_pool_size_one = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd default data pool replay window = 5 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon allow pool delete = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon cluster log file level = debug 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: debug asserts on shutdown = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon health detail to clog = false 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon host = 192.168.123.108 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon client directed command retry = 5 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: mon warn on pool no app = False 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: ms inject socket failures = 5000 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: fsid = 60776ecd-5e60-43ff-94d1-72ba986d0b0a 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout:[osd] 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd journal size = 100 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub load threshold = 5.0 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd scrub max interval = 600 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock profile = high_recovery_ops 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock skip benchmark = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd recover clone overlap = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd recovery max chunk = 1048576 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd debug shutdown = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd debug op order = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd debug verify stray on activate = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd debug trim objects = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd open classes on start = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd debug pg log writeout = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd deep scrub update digest min age = 30 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: osd map max advance = 10 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: journal zero on create = true 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: filestore ondisk finisher threads = 3 2026-03-23T19:30:37.953 INFO:teuthology.orchestra.run.vm08.stdout: filestore apply finisher threads = 3 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: bdev debug aio = true 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: osd debug misdirected ops = true 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: bdev async discard = True 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: bdev enable discard = True 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: bluestore block size = 96636764160 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: bluestore fsck on mount = True 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug bluefs = 1/20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug bluestore = 1/20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug osd = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug rocksdb = 4/10 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon osd backfillfull_ratio = 0.85 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon osd full ratio = 0.9 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon osd nearfull ratio = 0.8 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: osd failsafe full ratio = 0.95 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: osd mclock iops capacity threshold hdd = 49000 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: osd memory target = 1610612736 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: osd objectstore = bluestore 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout:[mgr] 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug mgr = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mgr/telemetry/nag = false 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout:[mon] 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug ms = 1 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug mon = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug paxos = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: debug auth = 20 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon data avail warn = 5 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon mgr mkfs grace = 240 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min pgs per osd = 4 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon osd reporter subtree level = osd 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon osd prime pg temp = true 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon reweight min bytes per osd = 10 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: auth mon ticket ttl = 660 # 11m 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: auth service ticket ttl = 240 # 4m 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: # don't complain about insecure global_id in the test suite 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: # 1m isn't quite enough 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon_down_mkfs_grace = 2m 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: mon_warn_on_filestore_osds = false 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout:[client] 2026-03-23T19:30:37.954 INFO:teuthology.orchestra.run.vm08.stdout: rgw cache enabled = true 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable ops log = true 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: rgw enable usage log = true 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: debug rbd = 20 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: debug rbd_mirror = 15 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: log to stderr = False 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: rbd default features = 125 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[mon.a] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.0] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.1] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.2] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.3] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.4] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.5] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.6] 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-03-23T19:30:37.955 INFO:teuthology.orchestra.run.vm08.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-03-23T19:30:37.963 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-03-23T19:30:37.963 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-03-23T19:30:38.005 INFO:tasks.ceph:Copying monmap to all nodes... 2026-03-23T19:30:38.005 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:38.005 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-03-23T19:30:38.061 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:38.061 DEBUG:teuthology.orchestra.run.vm08:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-03-23T19:30:38.118 INFO:tasks.ceph:Sending monmap to node ubuntu@vm04.local 2026-03-23T19:30:38.118 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:38.118 DEBUG:teuthology.orchestra.run.vm04:> sudo dd of=/etc/ceph/cluster2.keyring 2026-03-23T19:30:38.118 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-23T19:30:38.155 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:38.155 DEBUG:teuthology.orchestra.run.vm04:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:38.211 INFO:tasks.ceph:Sending monmap to node ubuntu@vm08.local 2026-03-23T19:30:38.211 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:38.211 DEBUG:teuthology.orchestra.run.vm08:> sudo dd of=/etc/ceph/cluster2.keyring 2026-03-23T19:30:38.211 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-03-23T19:30:38.243 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:38.243 DEBUG:teuthology.orchestra.run.vm08:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:38.299 INFO:tasks.ceph:Setting up mon nodes... 2026-03-23T19:30:38.299 INFO:tasks.ceph:Setting up mgr nodes... 2026-03-23T19:30:38.299 DEBUG:teuthology.orchestra.run.vm04:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-03-23T19:30:38.347 INFO:teuthology.orchestra.run.vm04.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-03-23T19:30:38.350 INFO:tasks.ceph:Setting up mds nodes... 2026-03-23T19:30:38.350 INFO:tasks.ceph_client:Setting up client nodes... 2026-03-23T19:30:38.350 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-03-23T19:30:38.391 INFO:teuthology.orchestra.run.vm04.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-03-23T19:30:38.404 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-03-23T19:30:38.439 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-03-23T19:30:38.449 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-03-23T19:30:38.524 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-03-23T19:30:38.535 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-03-23T19:30:38.609 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-03-23T19:30:38.621 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-03-23T19:30:38.696 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-03-23T19:30:38.709 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-03-23T19:30:38.787 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-03-23T19:30:38.801 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-03-23T19:30:38.881 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-03-23T19:30:38.894 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-03-23T19:30:38.973 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-03-23T19:30:38.986 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-03-23T19:30:39.066 INFO:teuthology.orchestra.run.vm08.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-03-23T19:30:39.078 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-03-23T19:30:39.078 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vm04.local'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@vm08.local'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-03-23T19:30:39.078 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-03-23T19:30:39.145 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:39.145 INFO:tasks.ceph:role: cluster2.osd.0 2026-03-23T19:30:39.145 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@vm08.local 2026-03-23T19:30:39.145 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:39.213 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:39.249 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-23T19:30:39.318 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@vm08.local -o noatime 2026-03-23T19:30:39.318 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-03-23T19:30:39.390 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-03-23T19:30:39.461 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-03-23T19:30:39.527 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:39.527 INFO:tasks.ceph:role: cluster2.osd.1 2026-03-23T19:30:39.527 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@vm08.local 2026-03-23T19:30:39.527 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:39.593 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:39.597 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-23T19:30:39.599 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@vm08.local -o noatime 2026-03-23T19:30:39.599 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-03-23T19:30:39.667 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-03-23T19:30:39.734 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-03-23T19:30:39.799 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-03-23T19:30:39.799 INFO:tasks.ceph:role: cluster2.osd.2 2026-03-23T19:30:39.799 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@vm08.local 2026-03-23T19:30:39.800 DEBUG:teuthology.orchestra.run.vm08:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=1310464 blks 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 attr=2, projid32bit=1 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout:data = bsize=4096 blocks=5241856, imaxpct=25 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout: = sunit=0 swidth=0 blks 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout:log =internal log bsize=4096 blocks=16384, version=2 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2026-03-23T19:30:39.865 INFO:teuthology.orchestra.run.vm08.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-03-23T19:30:39.869 INFO:teuthology.orchestra.run.vm08.stdout:Discarding blocks...Done. 2026-03-23T19:30:39.872 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@vm08.local -o noatime 2026-03-23T19:30:39.872 DEBUG:teuthology.orchestra.run.vm08:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-03-23T19:30:39.942 DEBUG:teuthology.orchestra.run.vm08:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-03-23T19:30:40.010 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:40.098 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.096+0000 7f3eeee20900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-03-23T19:30:40.098 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.096+0000 7f3eeee20900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-03-23T19:30:40.099 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.097+0000 7f3eeee20900 -1 bdev(0x555896fd7800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:40.099 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.097+0000 7f3eeee20900 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-03-23T19:30:40.717 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-03-23T19:30:40.743 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:40.823 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.822+0000 7fb083be0900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-03-23T19:30:40.824 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.822+0000 7fb083be0900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-03-23T19:30:40.824 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.822+0000 7fb083be0900 -1 bdev(0x5645bf7e1800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:40.824 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:40.822+0000 7fb083be0900 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-03-23T19:30:41.432 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-03-23T19:30:41.502 DEBUG:teuthology.orchestra.run.vm08:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:41.585 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:41.583+0000 7f18b526c900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-03-23T19:30:41.585 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:41.584+0000 7f18b526c900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-03-23T19:30:41.585 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:41.584+0000 7f18b526c900 -1 bdev(0x561686d63800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-03-23T19:30:41.585 INFO:teuthology.orchestra.run.vm08.stderr:2026-03-23T19:30:41.584+0000 7f18b526c900 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-03-23T19:30:42.224 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-03-23T19:30:42.253 INFO:tasks.ceph:Reading keys from all nodes... 2026-03-23T19:30:42.253 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:42.253 DEBUG:teuthology.orchestra.run.vm04:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-03-23T19:30:42.278 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.278 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-03-23T19:30:42.318 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.318 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-03-23T19:30:42.386 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.386 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-03-23T19:30:42.454 DEBUG:teuthology.orchestra.run.vm04:> set -ex 2026-03-23T19:30:42.454 DEBUG:teuthology.orchestra.run.vm04:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-03-23T19:30:42.470 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.470 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-03-23T19:30:42.510 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.510 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-03-23T19:30:42.567 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.567 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-03-23T19:30:42.622 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.622 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-03-23T19:30:42.678 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.679 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-03-23T19:30:42.733 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.733 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-03-23T19:30:42.790 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.790 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-03-23T19:30:42.847 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:42.847 DEBUG:teuthology.orchestra.run.vm08:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-03-23T19:30:42.904 INFO:tasks.ceph:Adding keys to all mons... 2026-03-23T19:30:42.904 DEBUG:teuthology.orchestra.run.vm08:> sudo tee -a /etc/ceph/cluster2.keyring 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[mgr.x] 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpxiakFBAA4zoahw2NnVRqDr7va/1m3g== 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[osd.0] 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDglMFp3vDCBRAAARGpyq/nagbdG/L1BXsu1w== 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[osd.1] 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDglMFpLTsDMRAAcvFjK9ZjrayJgqReAVHydA== 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[osd.2] 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDhlMFp+/PMIhAAYWINVd0gfEnHqCJjVewB5g== 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[client.0] 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpKWg6FxAAJtSkvcwIqIJhsoEdk5xcdA== 2026-03-23T19:30:42.969 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpubIgGhAA/9IE7x3cytI1h5hW1ilCwg== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.0] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpMh81HxAAtDrC/pIrMgFBQCx8ELQX3Q== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.1] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpOgZFJBAAXVczPc+EV5kF57XsFDKr/w== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.2] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpu+t1KRAAejIMfBMsn4WSxyPWSNzNYA== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.3] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFp3+PcLhAA9YPqHPoWVMsUJav2qdVE7Q== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.4] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpMXpyNBAAruDnedI4J28f0Gyb5815hw== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.5] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpSe3yORAA0lQYuS5+wkW5zPRo95vZkg== 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.6] 2026-03-23T19:30:42.970 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDflMFpJqvZAxAAAAE93cmsdMK9iTDk4ulpKg== 2026-03-23T19:30:42.971 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-03-23T19:30:43.059 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:43.104 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:43.150 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-03-23T19:30:43.197 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.245 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.294 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.344 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.392 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.444 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.496 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.544 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.592 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-03-23T19:30:43.641 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-03-23T19:30:43.641 DEBUG:teuthology.orchestra.run.vm08:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-03-23T19:30:43.668 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-03-23T19:30:43.768 DEBUG:teuthology.orchestra.run.vm08:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-03-23T19:30:43.796 DEBUG:teuthology.orchestra.run.vm08:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-03-23T19:30:43.852 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-03-23T19:30:43.852 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-03-23T19:30:43.852 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-03-23T19:30:43.896 INFO:tasks.ceph.cluster2.mon.a:Started 2026-03-23T19:30:43.896 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-03-23T19:30:43.896 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-03-23T19:30:43.896 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-03-23T19:30:43.898 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-03-23T19:30:43.898 DEBUG:tasks.ceph:set 0 configs 2026-03-23T19:30:43.898 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 config dump 2026-03-23T19:30:44.033 INFO:teuthology.orchestra.run.vm08.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-03-23T19:30:44.042 INFO:tasks.ceph:Setting crush tunables to default 2026-03-23T19:30:44.043 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 osd crush tunables default 2026-03-23T19:30:44.162 INFO:teuthology.orchestra.run.vm08.stderr:adjusted tunables profile to default 2026-03-23T19:30:44.173 INFO:tasks.ceph:check_enable_crimson: False 2026-03-23T19:30:44.173 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-03-23T19:30:44.173 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:44.173 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-03-23T19:30:44.198 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:44.198 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-03-23T19:30:44.265 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:30:44.265 DEBUG:teuthology.orchestra.run.vm08:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-03-23T19:30:44.331 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 osd new a86c779b-a4be-4ae9-878e-2afab05a4e94 0 2026-03-23T19:30:44.494 INFO:teuthology.orchestra.run.vm08.stdout:0 2026-03-23T19:30:44.504 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 osd new d124475d-dbc1-41f7-bfa1-4381afa83223 1 2026-03-23T19:30:44.628 INFO:teuthology.orchestra.run.vm08.stdout:1 2026-03-23T19:30:44.639 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 osd new c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d 2 2026-03-23T19:30:44.761 INFO:teuthology.orchestra.run.vm08.stdout:2 2026-03-23T19:30:44.773 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-03-23T19:30:44.773 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-03-23T19:30:44.774 INFO:tasks.ceph.cluster2.osd.0:Started 2026-03-23T19:30:44.774 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-03-23T19:30:44.774 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-03-23T19:30:44.776 INFO:tasks.ceph.cluster2.osd.1:Started 2026-03-23T19:30:44.776 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-03-23T19:30:44.776 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-03-23T19:30:44.777 INFO:tasks.ceph.cluster2.osd.2:Started 2026-03-23T19:30:44.777 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:30:44.833 INFO:tasks.ceph.cluster2.mgr.x.vm04.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-03-23T19:30:44.834 INFO:tasks.ceph.cluster2.mgr.x.vm04.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-03-23T19:30:44.834 INFO:tasks.ceph.cluster2.mgr.x.vm04.stderr: from numpy import show_config as show_numpy_config 2026-03-23T19:30:44.896 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:44.896 INFO:teuthology.orchestra.run.vm08.stdout:{"epoch":5,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:30:44.758967+0000","last_up_change":"0.000000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:44.907 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-03-23T19:30:44.907 INFO:tasks.ceph:Waiting for OSDs to come up 2026-03-23T19:30:44.947 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:30:44.945+0000 7fede59cf900 -1 Falling back to public interface 2026-03-23T19:30:44.950 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:30:44.947+0000 7f36f1419900 -1 Falling back to public interface 2026-03-23T19:30:44.961 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:30:44.959+0000 7effe5805900 -1 Falling back to public interface 2026-03-23T19:30:45.209 DEBUG:teuthology.orchestra.run.vm08:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:30:45.299 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:30:45.296+0000 7f36f1419900 -1 osd.0 0 log_to_monitors true 2026-03-23T19:30:45.306 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:30:45.304+0000 7fede59cf900 -1 osd.2 0 log_to_monitors true 2026-03-23T19:30:45.317 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:30:45.314+0000 7effe5805900 -1 osd.1 0 log_to_monitors true 2026-03-23T19:30:45.538 INFO:teuthology.misc.health.vm08.stdout: 2026-03-23T19:30:45.538 INFO:teuthology.misc.health.vm08.stdout:{"epoch":5,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:30:44.758967+0000","last_up_change":"0.000000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:45.546 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-03-23T19:30:47.082 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:30:47.080+0000 7f36ec380640 -1 osd.0 0 waiting for initial osdmap 2026-03-23T19:30:47.083 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:30:47.080+0000 7fede113b640 -1 osd.2 0 waiting for initial osdmap 2026-03-23T19:30:47.083 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:30:47.080+0000 7effe0f73640 -1 osd.1 0 waiting for initial osdmap 2026-03-23T19:30:47.093 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:30:47.091+0000 7f36e7997640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:47.093 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:30:47.091+0000 7effdc58a640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:47.093 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:30:47.091+0000 7feddc752640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-03-23T19:30:47.366 INFO:tasks.ceph.cluster2.mgr.x.vm04.stderr:2026-03-23T19:30:47.363+0000 7fdf74c0f640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-03-23T19:30:51.849 DEBUG:teuthology.orchestra.run.vm08:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:30:52.042 INFO:teuthology.misc.health.vm08.stdout: 2026-03-23T19:30:52.042 INFO:teuthology.misc.health.vm08.stdout:{"epoch":11,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:30:51.105963+0000","last_up_change":"2026-03-23T19:30:48.081660+0000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:48.364206+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6809","nonce":3072510388}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6811","nonce":3072510388}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6814","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6815","nonce":3072510388}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6813","nonce":3072510388}]},"public_addr":"192.168.123.108:6809/3072510388","cluster_addr":"192.168.123.108:6811/3072510388","heartbeat_back_addr":"192.168.123.108:6815/3072510388","heartbeat_front_addr":"192.168.123.108:6813/3072510388","state":["exists","up"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6817","nonce":326988043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6819","nonce":326988043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6822","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6823","nonce":326988043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6821","nonce":326988043}]},"public_addr":"192.168.123.108:6817/326988043","cluster_addr":"192.168.123.108:6819/326988043","heartbeat_back_addr":"192.168.123.108:6823/326988043","heartbeat_front_addr":"192.168.123.108:6821/326988043","state":["exists","up"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6801","nonce":1593614681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6803","nonce":1593614681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6807","nonce":1593614681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6805","nonce":1593614681}]},"public_addr":"192.168.123.108:6801/1593614681","cluster_addr":"192.168.123.108:6803/1593614681","heartbeat_back_addr":"192.168.123.108:6807/1593614681","heartbeat_front_addr":"192.168.123.108:6805/1593614681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.330219+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.267454+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.318092+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:52.052 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-03-23T19:30:52.052 INFO:tasks.ceph:Creating RBD pool 2026-03-23T19:30:52.052 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-03-23T19:30:53.116 INFO:teuthology.orchestra.run.vm08.stderr:pool 'rbd' created 2026-03-23T19:30:53.133 DEBUG:teuthology.orchestra.run.vm08:> rbd --cluster cluster2 pool init rbd 2026-03-23T19:30:56.135 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-03-23T19:30:56.136 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-03-23T19:30:56.136 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-03-23T19:30:56.342 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:56.356 INFO:teuthology.orchestra.run.vm08.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-03-23T19:30:56.356 INFO:tasks.ceph_manager:config epoch is 1 2026-03-23T19:30:56.356 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-03-23T19:30:56.356 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-03-23T19:30:56.356 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-03-23T19:30:56.606 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:56.619 INFO:teuthology.orchestra.run.vm08.stdout:{"epoch":5,"flags":0,"active_gid":4103,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6826","nonce":1009256750},{"type":"v1","addr":"192.168.123.104:6827","nonce":1009256750}]},"active_addr":"192.168.123.104:6827/1009256750","active_change":"2026-03-23T19:30:46.343305+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":3822686765}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":1259587296}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":2954789084}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"192.168.123.104:0","nonce":3720061776}]}]} 2026-03-23T19:30:56.620 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-03-23T19:30:56.620 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-03-23T19:30:56.620 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:30:56.842 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:56.842 INFO:teuthology.orchestra.run.vm08.stdout:{"epoch":15,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:30:56.122687+0000","last_up_change":"2026-03-23T19:30:48.081660+0000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:48.364206+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:52.255055+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6809","nonce":3072510388}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6811","nonce":3072510388}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6814","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6815","nonce":3072510388}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6813","nonce":3072510388}]},"public_addr":"192.168.123.108:6809/3072510388","cluster_addr":"192.168.123.108:6811/3072510388","heartbeat_back_addr":"192.168.123.108:6815/3072510388","heartbeat_front_addr":"192.168.123.108:6813/3072510388","state":["exists","up"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6817","nonce":326988043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6819","nonce":326988043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6822","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6823","nonce":326988043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6821","nonce":326988043}]},"public_addr":"192.168.123.108:6817/326988043","cluster_addr":"192.168.123.108:6819/326988043","heartbeat_back_addr":"192.168.123.108:6823/326988043","heartbeat_front_addr":"192.168.123.108:6821/326988043","state":["exists","up"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6801","nonce":1593614681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6803","nonce":1593614681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6807","nonce":1593614681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6805","nonce":1593614681}]},"public_addr":"192.168.123.108:6801/1593614681","cluster_addr":"192.168.123.108:6803/1593614681","heartbeat_back_addr":"192.168.123.108:6807/1593614681","heartbeat_front_addr":"192.168.123.108:6805/1593614681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.330219+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.267454+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.318092+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:56.857 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-03-23T19:30:56.857 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:30:57.068 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:30:57.068 INFO:teuthology.orchestra.run.vm08.stdout:{"epoch":15,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:30:56.122687+0000","last_up_change":"2026-03-23T19:30:48.081660+0000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:48.364206+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:52.255055+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6809","nonce":3072510388}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6811","nonce":3072510388}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6814","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6815","nonce":3072510388}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6813","nonce":3072510388}]},"public_addr":"192.168.123.108:6809/3072510388","cluster_addr":"192.168.123.108:6811/3072510388","heartbeat_back_addr":"192.168.123.108:6815/3072510388","heartbeat_front_addr":"192.168.123.108:6813/3072510388","state":["exists","up"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6817","nonce":326988043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6819","nonce":326988043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6822","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6823","nonce":326988043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6821","nonce":326988043}]},"public_addr":"192.168.123.108:6817/326988043","cluster_addr":"192.168.123.108:6819/326988043","heartbeat_back_addr":"192.168.123.108:6823/326988043","heartbeat_front_addr":"192.168.123.108:6821/326988043","state":["exists","up"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6801","nonce":1593614681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6803","nonce":1593614681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6807","nonce":1593614681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6805","nonce":1593614681}]},"public_addr":"192.168.123.108:6801/1593614681","cluster_addr":"192.168.123.108:6803/1593614681","heartbeat_back_addr":"192.168.123.108:6807/1593614681","heartbeat_front_addr":"192.168.123.108:6805/1593614681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.330219+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.267454+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.318092+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:30:57.082 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-03-23T19:30:57.082 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-03-23T19:30:57.082 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-03-23T19:30:57.197 INFO:teuthology.orchestra.run.vm08.stdout:34359738372 2026-03-23T19:30:57.197 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-23T19:30:57.199 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:30:57.199 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-23T19:30:57.206 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:30:57.206 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-23T19:30:57.428 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:30:57.445 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-03-23T19:30:57.487 INFO:teuthology.orchestra.run.vm08.stdout:34359738370 2026-03-23T19:30:57.499 INFO:teuthology.orchestra.run.vm08.stdout:34359738370 2026-03-23T19:30:57.503 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-03-23T19:30:57.513 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-03-23T19:30:58.446 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-23T19:30:58.504 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-23T19:30:58.514 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-23T19:30:58.668 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:30:58.682 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-03-23T19:30:58.737 INFO:teuthology.orchestra.run.vm08.stdout:34359738370 2026-03-23T19:30:58.751 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-03-23T19:30:58.765 INFO:teuthology.orchestra.run.vm08.stdout:34359738370 2026-03-23T19:30:58.778 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-03-23T19:30:59.683 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-03-23T19:30:59.752 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-03-23T19:30:59.779 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-03-23T19:30:59.914 INFO:teuthology.orchestra.run.vm08.stdout:34359738372 2026-03-23T19:30:59.935 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.1 2026-03-23T19:30:59.935 DEBUG:teuthology.parallel:result is None 2026-03-23T19:30:59.996 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:31:00.011 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.0 2026-03-23T19:31:00.011 DEBUG:teuthology.parallel:result is None 2026-03-23T19:31:00.029 INFO:teuthology.orchestra.run.vm08.stdout:34359738371 2026-03-23T19:31:00.042 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.2 2026-03-23T19:31:00.043 DEBUG:teuthology.parallel:result is None 2026-03-23T19:31:00.043 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-03-23T19:31:00.043 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:31:00.299 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:31:00.299 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:31:00.312 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-03-23T19:30:58.350129+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82096,"kb_used_data":1544,"kb_used_omap":23,"kb_used_meta":80424,"kb_avail":283033424,"statfs":{"total":289910292480,"available":289826226176,"internally_reserved":0,"allocated":1581056,"data_stored":1292025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24385,"internal_metadata":82354367},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.233823"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129636+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:30:56.129636+0000","last_peered":"2026-03-23T19:30:56.129636+0000","last_clean":"2026-03-23T19:30:56.129636+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:30:56.129636+0000","last_undegraded":"2026-03-23T19:30:56.129636+0000","last_fullsized":"2026-03-23T19:30:56.129636+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129656+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:30:56.129656+0000","last_peered":"2026-03-23T19:30:56.129656+0000","last_clean":"2026-03-23T19:30:56.129656+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:30:56.129656+0000","last_undegraded":"2026-03-23T19:30:56.129656+0000","last_fullsized":"2026-03-23T19:30:56.129656+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129897+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:30:56.129897+0000","last_peered":"2026-03-23T19:30:56.129897+0000","last_clean":"2026-03-23T19:30:56.129897+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:30:56.129897+0000","last_undegraded":"2026-03-23T19:30:56.129897+0000","last_fullsized":"2026-03-23T19:30:56.129897+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.130078+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:30:56.130078+0000","last_peered":"2026-03-23T19:30:56.130078+0000","last_clean":"2026-03-23T19:30:56.130078+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:30:56.130078+0000","last_undegraded":"2026-03-23T19:30:56.130078+0000","last_fullsized":"2026-03-23T19:30:56.130078+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.127375+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:30:56.127375+0000","last_peered":"2026-03-23T19:30:56.127375+0000","last_clean":"2026-03-23T19:30:56.127375+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:30:56.127375+0000","last_undegraded":"2026-03-23T19:30:56.127375+0000","last_fullsized":"2026-03-23T19:30:56.127375+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.261612+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:30:56.261612+0000","last_peered":"2026-03-23T19:30:56.261612+0000","last_clean":"2026-03-23T19:30:56.261612+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:30:56.261612+0000","last_undegraded":"2026-03-23T19:30:56.261612+0000","last_fullsized":"2026-03-23T19:30:56.261612+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.261624+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:30:56.261624+0000","last_peered":"2026-03-23T19:30:56.261624+0000","last_clean":"2026-03-23T19:30:56.261624+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:30:56.261624+0000","last_undegraded":"2026-03-23T19:30:56.261624+0000","last_fullsized":"2026-03-23T19:30:56.261624+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129967+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:30:56.129967+0000","last_peered":"2026-03-23T19:30:56.129967+0000","last_clean":"2026-03-23T19:30:56.129967+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:30:56.129967+0000","last_undegraded":"2026-03-23T19:30:56.129967+0000","last_fullsized":"2026-03-23T19:30:56.129967+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'92","reported_seq":134,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129617+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:30:56.129617+0000","last_peered":"2026-03-23T19:30:56.129617+0000","last_clean":"2026-03-23T19:30:56.129617+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:30:56.129617+0000","last_undegraded":"2026-03-23T19:30:56.129617+0000","last_fullsized":"2026-03-23T19:30:56.129617+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":122880,"data_stored":34713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8127,"internal_metadata":27451457},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7477,"internal_metadata":27452107},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8781,"internal_metadata":27450803},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:31:00.312 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:31:00.527 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:31:00.527 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:31:00.540 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-03-23T19:31:00.350354+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82096,"kb_used_data":1552,"kb_used_omap":24,"kb_used_meta":80423,"kb_avail":283033424,"statfs":{"total":289910292480,"available":289826226176,"internally_reserved":0,"allocated":1589248,"data_stored":1295581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25035,"internal_metadata":82353717},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.234049"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129636+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:30:56.129636+0000","last_peered":"2026-03-23T19:30:56.129636+0000","last_clean":"2026-03-23T19:30:56.129636+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:30:56.129636+0000","last_undegraded":"2026-03-23T19:30:56.129636+0000","last_fullsized":"2026-03-23T19:30:56.129636+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129656+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:30:56.129656+0000","last_peered":"2026-03-23T19:30:56.129656+0000","last_clean":"2026-03-23T19:30:56.129656+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:30:56.129656+0000","last_undegraded":"2026-03-23T19:30:56.129656+0000","last_fullsized":"2026-03-23T19:30:56.129656+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129897+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:30:56.129897+0000","last_peered":"2026-03-23T19:30:56.129897+0000","last_clean":"2026-03-23T19:30:56.129897+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:30:56.129897+0000","last_undegraded":"2026-03-23T19:30:56.129897+0000","last_fullsized":"2026-03-23T19:30:56.129897+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.130078+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:30:56.130078+0000","last_peered":"2026-03-23T19:30:56.130078+0000","last_clean":"2026-03-23T19:30:56.130078+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:30:56.130078+0000","last_undegraded":"2026-03-23T19:30:56.130078+0000","last_fullsized":"2026-03-23T19:30:56.130078+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.127375+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:30:56.127375+0000","last_peered":"2026-03-23T19:30:56.127375+0000","last_clean":"2026-03-23T19:30:56.127375+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:30:56.127375+0000","last_undegraded":"2026-03-23T19:30:56.127375+0000","last_fullsized":"2026-03-23T19:30:56.127375+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.261612+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:30:56.261612+0000","last_peered":"2026-03-23T19:30:56.261612+0000","last_clean":"2026-03-23T19:30:56.261612+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:30:56.261612+0000","last_undegraded":"2026-03-23T19:30:56.261612+0000","last_fullsized":"2026-03-23T19:30:56.261612+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.261624+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:30:56.261624+0000","last_peered":"2026-03-23T19:30:56.261624+0000","last_clean":"2026-03-23T19:30:56.261624+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:30:56.261624+0000","last_undegraded":"2026-03-23T19:30:56.261624+0000","last_fullsized":"2026-03-23T19:30:56.261624+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129967+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:30:56.129967+0000","last_peered":"2026-03-23T19:30:56.129967+0000","last_clean":"2026-03-23T19:30:56.129967+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:30:56.129967+0000","last_undegraded":"2026-03-23T19:30:56.129967+0000","last_fullsized":"2026-03-23T19:30:56.129967+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'92","reported_seq":134,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-03-23T19:30:56.129617+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:30:56.129617+0000","last_peered":"2026-03-23T19:30:56.129617+0000","last_clean":"2026-03-23T19:30:56.129617+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:30:56.129617+0000","last_undegraded":"2026-03-23T19:30:56.129617+0000","last_fullsized":"2026-03-23T19:30:56.129617+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":26976,"kb_used_data":128,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344864,"statfs":{"total":96636764160,"available":96609140736,"internally_reserved":0,"allocated":131072,"data_stored":38269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8777,"internal_metadata":27450807},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26808,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7477,"internal_metadata":27452107},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27560,"kb_used_data":712,"kb_used_omap":8,"kb_used_meta":26807,"kb_avail":94344280,"statfs":{"total":96636764160,"available":96608542720,"internally_reserved":0,"allocated":729088,"data_stored":628656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8781,"internal_metadata":27450803},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:31:00.540 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-03-23T19:31:00.540 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-03-23T19:31:00.540 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-03-23T19:31:00.540 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-03-23T19:31:00.791 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:31:00.791 INFO:teuthology.orchestra.run.vm08.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-03-23T19:31:00.803 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-03-23T19:31:00.806 INFO:teuthology.run_tasks:Running task exec... 2026-03-23T19:31:00.812 INFO:teuthology.task.exec:Executing custom commands... 2026-03-23T19:31:00.812 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@vm08.local 2026-03-23T19:31:00.812 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:01.089 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.4] 2026-03-23T19:31:01.089 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFpUas4MhAApYakVjh/q43RE8ANhtlJDg== 2026-03-23T19:31:01.090 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:01.090 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:01.090 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.4 2026-03-23T19:31:01.106 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:01.392 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.5] 2026-03-23T19:31:01.392 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFpiX7GNxAAsNnkjn4sctK9b4jXGrHCoA== 2026-03-23T19:31:01.392 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:01.392 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:01.392 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.5 2026-03-23T19:31:01.407 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:01.782 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.6] 2026-03-23T19:31:01.783 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDHlMFpDpLdARAAiXITMsa+pN+tkaPDCCiEmQ== 2026-03-23T19:31:01.783 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:01.783 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:01.783 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.6 2026-03-23T19:31:01.796 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:02.057 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.4] 2026-03-23T19:31:02.057 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpMXpyNBAAruDnedI4J28f0Gyb5815hw== 2026-03-23T19:31:02.057 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:02.057 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:02.057 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.4 2026-03-23T19:31:02.070 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:02.339 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.5] 2026-03-23T19:31:02.339 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpSe3yORAA0lQYuS5+wkW5zPRo95vZkg== 2026-03-23T19:31:02.339 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:02.339 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:02.339 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.5 2026-03-23T19:31:02.353 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:02.618 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.6] 2026-03-23T19:31:02.618 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDflMFpJqvZAxAAAAE93cmsdMK9iTDk4ulpKg== 2026-03-23T19:31:02.618 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:02.618 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:02.618 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.6 2026-03-23T19:31:02.633 INFO:teuthology.run_tasks:Running task exec... 2026-03-23T19:31:02.635 INFO:teuthology.task.exec:Executing custom commands... 2026-03-23T19:31:02.635 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@vm08.local 2026-03-23T19:31:02.635 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:02.919 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror] 2026-03-23T19:31:02.919 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFpZW+sGBAA5mcL98kqSmiBPO9Bu3Yr0A== 2026-03-23T19:31:02.919 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror-peer" 2026-03-23T19:31:02.919 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:02.919 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror 2026-03-23T19:31:02.932 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:03.197 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.0] 2026-03-23T19:31:03.197 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFpNuJxHhAAzcH0xucKl4YuR7btCgw5qQ== 2026-03-23T19:31:03.197 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:03.197 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:03.197 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.0 2026-03-23T19:31:03.212 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:03.645 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.1] 2026-03-23T19:31:03.645 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFptMmbIRAAks9OeSQSbJxNIbGhvQWUuw== 2026-03-23T19:31:03.645 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:03.645 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:03.645 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.1 2026-03-23T19:31:03.659 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:03.986 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.2] 2026-03-23T19:31:03.986 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFpckg0JxAA3UGNqXgqx5mg5l2eI7b9HA== 2026-03-23T19:31:03.986 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:03.986 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:03.986 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.2 2026-03-23T19:31:04.001 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:04.265 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.3] 2026-03-23T19:31:04.266 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDGlMFp19WpLBAA6cw8eAcg6P810Dl1fNqJLQ== 2026-03-23T19:31:04.266 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:04.266 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:04.266 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.3 2026-03-23T19:31:04.279 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:04.546 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror] 2026-03-23T19:31:04.546 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpubIgGhAA/9IE7x3cytI1h5hW1ilCwg== 2026-03-23T19:31:04.546 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror-peer" 2026-03-23T19:31:04.546 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:04.546 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror 2026-03-23T19:31:04.560 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:04.816 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.0] 2026-03-23T19:31:04.816 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpMh81HxAAtDrC/pIrMgFBQCx8ELQX3Q== 2026-03-23T19:31:04.816 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:04.816 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:04.816 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.0 2026-03-23T19:31:04.830 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:05.093 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.1] 2026-03-23T19:31:05.093 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpOgZFJBAAXVczPc+EV5kF57XsFDKr/w== 2026-03-23T19:31:05.093 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:05.093 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:05.093 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.1 2026-03-23T19:31:05.108 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:05.378 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.2] 2026-03-23T19:31:05.378 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFpu+t1KRAAejIMfBMsn4WSxyPWSNzNYA== 2026-03-23T19:31:05.378 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:05.378 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:05.378 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.2 2026-03-23T19:31:05.392 DEBUG:teuthology.orchestra.run.vm08:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-03-23T19:31:05.648 INFO:teuthology.orchestra.run.vm08.stdout:[client.mirror.3] 2026-03-23T19:31:05.648 INFO:teuthology.orchestra.run.vm08.stdout: key = AQDelMFp3+PcLhAA9YPqHPoWVMsUJav2qdVE7Q== 2026-03-23T19:31:05.648 INFO:teuthology.orchestra.run.vm08.stdout: caps mon = "profile rbd-mirror" 2026-03-23T19:31:05.648 INFO:teuthology.orchestra.run.vm08.stdout: caps osd = "profile rbd" 2026-03-23T19:31:05.648 INFO:teuthology.orchestra.run.vm08.stderr:updated caps for client.mirror.3 2026-03-23T19:31:05.662 INFO:teuthology.run_tasks:Running task workunit... 2026-03-23T19:31:05.666 INFO:tasks.workunit:Pulling workunits from ref 0392f78529848ec72469e8e431875cb98d3a5fb4 2026-03-23T19:31:05.666 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-03-23T19:31:05.666 DEBUG:teuthology.orchestra.run.vm08:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-23T19:31:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:31:05.682 INFO:teuthology.orchestra.run.vm08.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-03-23T19:31:05.682 DEBUG:teuthology.orchestra.run.vm08:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-23T19:31:05.738 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-23T19:31:05.738 DEBUG:teuthology.orchestra.run.vm08:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-03-23T19:31:05.794 INFO:tasks.workunit:timeout=3h 2026-03-23T19:31:05.794 INFO:tasks.workunit:cleanup=True 2026-03-23T19:31:05.794 DEBUG:teuthology.orchestra.run.vm08:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/kshtsk/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout 0392f78529848ec72469e8e431875cb98d3a5fb4 2026-03-23T19:31:05.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Note: switching to '0392f78529848ec72469e8e431875cb98d3a5fb4'. 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:changes and commit them, and you can discard any commits you make in this 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:state without impacting any branches by switching back to a branch. 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:If you want to create a new branch to retain commits you create, you may 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:do so (now or later) by using -c with the switch command. Example: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: git switch -c 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Or undo this operation with: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: git switch - 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:31:50.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:HEAD is now at 0392f785298 qa/tasks/keystone: restart mariadb for rocky and alma linux too 2026-03-23T19:31:50.923 DEBUG:teuthology.orchestra.run.vm08:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-03-23T19:31:50.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-03-23T19:31:50.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-03-23T19:31:50.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-03-23T19:31:51.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-03-23T19:31:51.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-03-23T19:31:51.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-03-23T19:31:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-03-23T19:31:51.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-03-23T19:31:51.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-03-23T19:31:51.128 DEBUG:teuthology.orchestra.run.vm08:> set -ex 2026-03-23T19:31:51.128 DEBUG:teuthology.orchestra.run.vm08:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-03-23T19:31:51.185 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror.sh on cluster1.client.mirror... 2026-03-23T19:31:51.186 INFO:tasks.workunit:Running workunit rbd/rbd_mirror.sh... 2026-03-23T19:31:51.186 DEBUG:teuthology.orchestra.run.vm08:workunit test rbd/rbd_mirror.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0392f78529848ec72469e8e431875cb98d3a5fb4 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_IMAGE_FEATURES=layering,exclusive-lock RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-03-23T19:31:51.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -gt 0 ']' 2026-03-23T19:31:51.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot '!=' snapshot ']' 2026-03-23T19:31:51.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-03-23T19:31:51.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ RBD_MIRROR_INSTANCES=4 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CLUSTER1=cluster1 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CLUSTER2=cluster2 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ PEER_CLUSTER_SUFFIX= 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ POOL=mirror 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ PARENT_POOL=mirror_parent 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ NS1=ns1 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ NS2=ns2 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ TEMPDIR= 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ID=mirror 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ RBD_MIRROR_MODE=snapshot 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ MIRROR_POOL_MODE=pool 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' snapshot = snapshot ']' 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ MIRROR_POOL_MODE=image 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS='--id mirror' 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ LAST_MIRROR_INSTANCE=3 2026-03-23T19:31:51.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-03-23T19:31:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-03-23T19:31:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-03-23T19:31:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_BIN=. 2026-03-23T19:31:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_SRC=. 2026-03-23T19:31:51.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' -e CMakeCache.txt ']' 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 -gt 0 ']' 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ setup 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local c 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ setup_tempdir 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:31:51.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mktemp -d 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ TEMPDIR=/tmp/tmp.ZXt03XlClS 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z 1 ']' 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ setup_pools cluster1 cluster2 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster2 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mon_map_file 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mon_addr 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local admin_key_file 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:51.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-03-23T19:31:51.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror' already exists 2026-03-23T19:31:51.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:51.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-03-23T19:31:52.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror_parent' already exists 2026-03-23T19:31:52.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:52.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 pool init mirror 2026-03-23T19:31:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:56.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-03-23T19:31:59.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 1 ']' 2026-03-23T19:31:59.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-03-23T19:31:59.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:59.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1-DNE mirror image 2026-03-23T19:31:59.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-03-23T19:31:59.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-03-23T19:31:59.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-03-23T19:31:59.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-03-23T19:31:59.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-03-23T19:31:59.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-03-23T19:31:59.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-03-23T19:31:59.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z ']' 2026-03-23T19:31:59.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z 1 ']' 2026-03-23T19:31:59.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mon_map_file=/tmp/tmp.ZXt03XlClS/cluster2.monmap 2026-03-23T19:31:59.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:59.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 mon getmap 2026-03-23T19:31:59.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:got monmap epoch 1 2026-03-23T19:31:59.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ monmaptool --print /tmp/tmp.ZXt03XlClS/cluster2.monmap 2026-03-23T19:31:59.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -E 'mon\.' 2026-03-23T19:31:59.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ head -n 1 2026-03-23T19:31:59.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -E 's/\/[0-9]+//g' 2026-03-23T19:31:59.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-03-23T19:31:59.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mon_addr='[v2:192.168.123.108:3300,v1:192.168.123.108:6789]' 2026-03-23T19:31:59.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_key_file=/tmp/tmp.ZXt03XlClS/cluster2.client.mirror.key 2026-03-23T19:31:59.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:59.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 auth get-key client.mirror 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_add cluster1 mirror client.mirror@cluster2-DNE '' --remote-mon-host '[v2:192.168.123.108:3300,v1:192.168.123.108:6789]' --remote-key-file /tmp/tmp.ZXt03XlClS/cluster2.client.mirror.key 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:31:59.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster2-DNE 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid_var_name 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local error_code 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local peer_uuid 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 16 32 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:31:59.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror client.mirror@cluster2-DNE --remote-mon-host '[v2:192.168.123.108:3300,v1:192.168.123.108:6789]' --remote-key-file /tmp/tmp.ZXt03XlClS/cluster2.client.mirror.key 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_uuid=04da4782-4604-444b-8d54-7d24cb321a84 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 17 ']' 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 -eq 0 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:00.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_add cluster1 mirror_parent client.mirror@cluster2-DNE uuid 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster2-DNE 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid_var_name 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n uuid ']' 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ uuid_var_name=uuid 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local error_code 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local peer_uuid 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 16 32 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:32:00.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent client.mirror@cluster2-DNE 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_uuid=a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 17 ']' 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 -eq 0 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n uuid ']' 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval uuid=a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ uuid=a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:00.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 mon-host '[v2:192.168.123.108:3300,v1:192.168.123.108:6789]' 2026-03-23T19:32:00.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:00.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 key-file /tmp/tmp.ZXt03XlClS/cluster2.client.mirror.key 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ setup_pools cluster2 cluster1 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster1 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mon_map_file 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mon_addr 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local admin_key_file 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:00.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-03-23T19:32:00.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror' already exists 2026-03-23T19:32:00.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:00.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-03-23T19:32:01.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror_parent' already exists 2026-03-23T19:32:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:01.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 pool init mirror 2026-03-23T19:32:04.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:04.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-03-23T19:32:07.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 1 ']' 2026-03-23T19:32:07.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-03-23T19:32:07.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:07.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2-DNE mirror image 2026-03-23T19:32:07.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-03-23T19:32:07.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-03-23T19:32:07.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-03-23T19:32:07.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-03-23T19:32:07.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-03-23T19:32:07.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-03-23T19:32:07.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-03-23T19:32:07.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z ']' 2026-03-23T19:32:07.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z 1 ']' 2026-03-23T19:32:07.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mon_map_file=/tmp/tmp.ZXt03XlClS/cluster1.monmap 2026-03-23T19:32:07.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:07.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 mon getmap 2026-03-23T19:32:08.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:got monmap epoch 1 2026-03-23T19:32:08.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ monmaptool --print /tmp/tmp.ZXt03XlClS/cluster1.monmap 2026-03-23T19:32:08.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -E 'mon\.' 2026-03-23T19:32:08.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ head -n 1 2026-03-23T19:32:08.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -E 's/\/[0-9]+//g' 2026-03-23T19:32:08.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-03-23T19:32:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mon_addr='[v2:192.168.123.104:3300,v1:192.168.123.104:6789]' 2026-03-23T19:32:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_key_file=/tmp/tmp.ZXt03XlClS/cluster1.client.mirror.key 2026-03-23T19:32:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 auth get-key client.mirror 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_add cluster2 mirror client.mirror@cluster1-DNE '' --remote-mon-host '[v2:192.168.123.104:3300,v1:192.168.123.104:6789]' --remote-key-file /tmp/tmp.ZXt03XlClS/cluster1.client.mirror.key 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster1-DNE 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid_var_name 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local error_code 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local peer_uuid 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 16 32 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:32:08.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror client.mirror@cluster1-DNE --remote-mon-host '[v2:192.168.123.104:3300,v1:192.168.123.104:6789]' --remote-key-file /tmp/tmp.ZXt03XlClS/cluster1.client.mirror.key 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_uuid=701534f2-a033-4b7c-b240-20580a897329 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 17 ']' 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 -eq 0 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_add cluster2 mirror_parent client.mirror@cluster1-DNE uuid 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_cluster=cluster1-DNE 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local uuid_var_name 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n uuid ']' 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ uuid_var_name=uuid 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local error_code 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local peer_uuid 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 16 32 2026-03-23T19:32:08.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:32:08.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent client.mirror@cluster1-DNE 2026-03-23T19:32:08.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ peer_uuid=5b860b55-e43a-495c-bd9f-ac730592f7f5 2026-03-23T19:32:08.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 17 ']' 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 -eq 0 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n uuid ']' 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval uuid=5b860b55-e43a-495c-bd9f-ac730592f7f5 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ uuid=5b860b55-e43a-495c-bd9f-ac730592f7f5 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:08.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent 5b860b55-e43a-495c-bd9f-ac730592f7f5 mon-host '[v2:192.168.123.104:3300,v1:192.168.123.104:6789]' 2026-03-23T19:32:08.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:32:08.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent 5b860b55-e43a-495c-bd9f-ac730592f7f5 key-file /tmp/tmp.ZXt03XlClS/cluster1.client.mirror.key 2026-03-23T19:32:08.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:08.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: mirror from default namespace to non-default namespace' 2026-03-23T19:32:08.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:32:08.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:32:08.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:32:08 TEST: mirror from default namespace to non-default namespace 2026-03-23T19:32:08.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:32:08 TEST: mirror from default namespace to non-default namespace 2026-03-23T19:32:08.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:32:08.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:08.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:32:08.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:08.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:08.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:08.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:08.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:08.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:08.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image=test 2026-03-23T19:32:08.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror init-only 2026-03-23T19:32:08.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror init-only 2026-03-23T19:32:08.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns1 2026-03-23T19:32:08.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns1 2026-03-23T19:32:08.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror image --remote-namespace ns1 2026-03-23T19:32:08.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image --remote-namespace '' 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:08.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test 2026-03-23T19:32:08.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:32:08.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:08.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:32:08.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:08.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-03-23T19:32:09.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:32:09.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:32:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:32:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:32:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 10cae872e5e 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.10cae872e5e 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:32:08 2026 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:32:08 2026 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:32:08 2026 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: d3daa918-bb97-4c84-9c6c-9270540105de 2026-03-23T19:32:09.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:32:09.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:09.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:32:10.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:32:10.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:10.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:10.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:10.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:10.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:10.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:10.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:10.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:10.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:10.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:10.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:10.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:10.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:10.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:10.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:10.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:10.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:10.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:10.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:10.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:10.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:10.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:10.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:10.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:10.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:10.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:10.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:10.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:10.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:10.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:10.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:10.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:10.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:32:11.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:11.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:11.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:11.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:11.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:11.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:11.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:11.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:11.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:11.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:11.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:11.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:11.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:11.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:11.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:11.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:11.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:11.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:11.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:11.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:11.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:11.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:11.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:11.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:11.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:12.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:12.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:12.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:12.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:12.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:12.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:12.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:12.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:12.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:12.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:32:14.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:14.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:14.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:14.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:14.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:14.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:14.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:14.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:14.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:14.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:14.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:14.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:14.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:14.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:14.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:14.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:14.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:14.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:14.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:14.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:14.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:14.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:14.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:14.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:14.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:14.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:14.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:14.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:14.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:14.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:14.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:14.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:14.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:14.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:14.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:14.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:14.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:14.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 4 2026-03-23T19:32:18.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:18.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:18.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:18.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:18.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:18.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:18.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:18.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:18.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:18.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:18.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:18.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:18.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:18.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:18.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:18.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:18.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:18.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:18.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:18.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:18.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:18.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:18.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:18.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:18.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:18.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:18.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:18.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:18.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:18.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:18.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:18.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:18.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-03-23T19:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 8 2026-03-23T19:32:26.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:26.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:26.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:26.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:26.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:26.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:32:26.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:32:26.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:32:26.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:32:26.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:32:26.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:32:26.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:32:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:32:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 3225.8 bytes/sec: 13 MiB/s 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror test 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:26.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:32:27.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 4 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror test 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:27.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:32:27.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:32:27.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:27.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:27.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-03-23T19:32:27.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test.status 2026-03-23T19:32:27.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:32:27.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:27.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:32:27.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=4 2026-03-23T19:32:27.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:32:27.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:27.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-03-23T19:32:27.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:27.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:27.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=3 2026-03-23T19:32:27.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 4 = 3 2026-03-23T19:32:27.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:27.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:32:28.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-03-23T19:32:28.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:28.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=3 2026-03-23T19:32:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 4 = 3 2026-03-23T19:32:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:28.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:32:28.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:28.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:28.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:28.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:28.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:32:28.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-03-23T19:32:28.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:28.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:28.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=4 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 4 = 4 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:29.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:29.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: d3daa918-bb97-4c84-9c6c-9270540105de 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294329,"replay_state":"idle"} 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:20' 2026-03-23T19:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:32:30.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:32:30.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:32:30.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294329,"replay_state":"idle"} 2026-03-23T19:32:30.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:30.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:30.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:32:30.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:30.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:32:30.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:32:30.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:32:30.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:20' 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-03-23 19:32:20' 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:32:20' 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:30.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-03-23T19:32:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING142734278mirror.0vm08.local20.2.0-712-g70f8415btrueOK42804283mirror.1vm08.local20.2.0-712-g70f8415bfalseOK42874294mirror.2vm08.local20.2.0-712-g70f8415bfalseOK42994305mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestd3daa918-bb97-4c84-9c6c-9270540105deup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294329,"replay_state":"idle"}42734278mirror.0vm08.local2026-03-23 19:32:20' 2026-03-23T19:32:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:32:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:32:30.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:20' 2026-03-23T19:32:30.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:32:30.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror test 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:32:30.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:30.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:32:30.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-03-23T19:32:30.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test.export 2026-03-23T19:32:30.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:30.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:32:30.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:32:30.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:32:30.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:30.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:32:30.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:30.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:32:30.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:30.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:30.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test 2026-03-23T19:32:30.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:30.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:30.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:30.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-03-23T19:32:30.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-03-23T19:32:31.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:32:31.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-03-23T19:32:31.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:32:31.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:32:31.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:32:31.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:32:31.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:32:31.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-03-23T19:32:31.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:32:31.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:32:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:32:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:32:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns1 2026-03-23T19:32:32.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror init-only 2026-03-23T19:32:32.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: mirror from non-default namespace to default namespace' 2026-03-23T19:32:32.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:32:32.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:32:32.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:32:32 TEST: mirror from non-default namespace to default namespace 2026-03-23T19:32:32.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:32:32 TEST: mirror from non-default namespace to default namespace 2026-03-23T19:32:32.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image --remote-namespace '' 2026-03-23T19:32:32.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror image --remote-namespace ns1 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns1 test 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:32.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns1/test 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-03-23T19:32:32.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:32:32.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 10e75ca1b876 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.10e75ca1b876 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:32:32 2026 2026-03-23T19:32:32.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:32:32 2026 2026-03-23T19:32:32.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:32:32 2026 2026-03-23T19:32:32.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:32:32.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:32:32.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: e14d2f14-e1a5-4e25-9be6-dbe9136b4729 2026-03-23T19:32:32.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:32.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:33.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:32:33.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:32:33.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:33.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:33.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:33.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:33.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:33.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:33.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:33.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:33.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:33.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:33.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:33.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:33.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:33.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:33.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:33.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:33.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:32:33.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:33.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:33.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:33.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:33.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:34.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:34.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:34.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:34.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:32:34.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:34.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:34.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:34.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:32:34.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:34.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:34.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:34.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:34.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:32:35.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:35.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:35.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:35.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:35.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:35.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:35.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:35.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:35.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:35.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:35.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:35.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:35.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:35.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:32:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:35.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:35.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:35.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:35.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:35.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:35.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:35.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:32:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:35.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:35.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:35.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:35.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:35.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:35.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:35.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:32:35.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:35.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:35.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:35.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:35.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:32:37.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:37.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:37.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:37.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:37.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:37.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:37.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:37.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:37.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:32:37.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:32:37.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:32:37.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:32:37.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:32:37.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:32:37.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:32:37.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:32:37.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2702.7 bytes/sec: 11 MiB/s 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror/ns1 test 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:37.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-03-23T19:32:37.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 6 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror/ns1 test 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-03-23T19:32:37.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test.status 2026-03-23T19:32:37.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:32:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:32:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:32:37.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-03-23T19:32:37.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:37.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:32:37.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=6 2026-03-23T19:32:37.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:32:37.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:37.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:32:38.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:38.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=6 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 6 = 6 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:39.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:32:39.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:39.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:32:39.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:32:39.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: e14d2f14-e1a5-4e25-9be6-dbe9136b4729 2026-03-23T19:32:39.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:32:39.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294352,"replay_state":"idle"} 2026-03-23T19:32:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:32:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:35' 2026-03-23T19:32:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:32:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:32:39.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:32:39.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294352,"replay_state":"idle"} 2026-03-23T19:32:39.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:39.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:39.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:32:39.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:39.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:32:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:32:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:32:39.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:32:39.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:35' 2026-03-23T19:32:39.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:32:35' 2026-03-23T19:32:39.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:39.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:32:35' 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:39.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:32:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING142734278mirror.0vm08.local20.2.0-712-g70f8415btrueOK42804283mirror.1vm08.local20.2.0-712-g70f8415bfalseOK42874294mirror.2vm08.local20.2.0-712-g70f8415bfalseOK42994305mirror.3vm08.local20.2.0-712-g70f8415bfalseOKteste14d2f14-e1a5-4e25-9be6-dbe9136b4729up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294352,"replay_state":"idle"}42734278mirror.0vm08.local2026-03-23 19:32:35' 2026-03-23T19:32:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:32:39.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:32:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:35' 2026-03-23T19:32:39.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:32:39.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:32:39.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:39.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror/ns1 test 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-03-23T19:32:39.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test.export 2026-03-23T19:32:39.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:39.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export 2026-03-23T19:32:39.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:32:39.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:32:39.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:39.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export 2026-03-23T19:32:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:39.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:39.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:39.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:39.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns1 test 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:39.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-03-23T19:32:39.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-03-23T19:32:40.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:40.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:32:40.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:32:40.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:32:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:32:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:32:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:32:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:32:40.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:32:40.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:32:40.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:32:40.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:40.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:32:40.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:40.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror init-only 2026-03-23T19:32:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns1 2026-03-23T19:32:40.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror image 2026-03-23T19:32:40.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror image 2026-03-23T19:32:40.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-03-23T19:32:40.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-03-23T19:32:40.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: add image and test replay' 2026-03-23T19:32:40.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:32:40.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:32:40 TEST: add image and test replay 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:32:40 TEST: add image and test replay 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:40.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:32:40.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:40.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:40.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-03-23T19:32:41.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:32:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 11092a1e7879 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.11092a1e7879 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:32:41.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_image_meta cluster2 mirror test key1 value1 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local key=key1 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=value1 2026-03-23T19:32:41.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key1 value1 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_image_meta cluster2 mirror test key2 value2 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local key=key2 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=value2 2026-03-23T19:32:41.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key2 value2 2026-03-23T19:32:41.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:32:41.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:41.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:41.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:42.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:32:42.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:42.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:42.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:42.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:42.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:42.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:42.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:42.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:42.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:42.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:42.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:32:42.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:42.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:42.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:42.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:42.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:32:42.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:42.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:42.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:42.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:42.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:42.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:42.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:42.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:42.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:32:42.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:42.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:42.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:42.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:42.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:32:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:43.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:43.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:43.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:43.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:43.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:43.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:43.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:43.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:43.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:43.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:43.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:32:43.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:43.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:43.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:43.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:43.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:43.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:43.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:43.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:32:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:43.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:43.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:43.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:43.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:32:44.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:32:44.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:44.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:44.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:44.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:32:46.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:46.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:46.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:46.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:46.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:46.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:46.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:46.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:32:46.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:32:46.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:32:46.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:32:46.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:32:46.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:32:46.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:32:46.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:32:46.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:32:46.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:32:46.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 8 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:32:46.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:46.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:46.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:32:46.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:32:46.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:32:46.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:46.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:32:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=8 2026-03-23T19:32:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:32:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:32:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:46.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:46.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:46.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:32:46.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:46.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=7 2026-03-23T19:32:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 8 = 7 2026-03-23T19:32:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:32:47.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:47.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:32:47.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:47.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:32:47.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:32:47.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:32:47.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:32:47.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 8 = '' 2026-03-23T19:32:47.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:47.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:32:47.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:48.009 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=8 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 8 = 8 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:48.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:49.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:49.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:49.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:49.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:49.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294361,"replay_state":"idle"} 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:45' 2026-03-23T19:32:49.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:32:49.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:32:49.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:32:49.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294361,"replay_state":"idle"} 2026-03-23T19:32:49.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:49.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:49.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:32:49.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:49.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:32:49.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:32:49.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:32:49.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:45' 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:32:45' 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:32:45' 2026-03-23T19:32:49.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:49.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:49.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:32:49.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING142734278mirror.0vm08.local20.2.0-712-g70f8415btrueOK42804283mirror.1vm08.local20.2.0-712-g70f8415bfalseOK42874294mirror.2vm08.local20.2.0-712-g70f8415bfalseOK42994305mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294361,"replay_state":"idle"}42734278mirror.0vm08.local2026-03-23 19:32:45' 2026-03-23T19:32:49.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:32:49.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:32:49.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:45' 2026-03-23T19:32:49.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:32:49.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:32:49.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:32:49.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:32:49.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:32:49.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:49.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:49.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test down+unknown 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:49.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test down+unknown '' '' 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:50.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294361,"replay_state":"idle"} 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:45 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 7 .mirror.primary.b39a943d-a4da-408f-871a-d06e6210cd11.011a0bc1-37ad-4b19-a2f6-052f6ef975da (peer_uuids:[]) 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 8 .mirror.primary.b39a943d-a4da-408f-871a-d06e6210cd11.eab2845e-ea1e-4e2b-aa0b-c89382c044c4 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:32:50.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*down+unknown' 2026-03-23T19:32:50.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: down+unknown 2026-03-23T19:32:50.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:32:50.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: status not found 2026-03-23T19:32:50.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:50.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo down+unknown 2026-03-23T19:32:50.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:32:50.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: ' 2026-03-23T19:32:50.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:32:50.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update= 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test down+unknown '' 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local prev_last_update= 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:32:50.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:32:50.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='UNKNOWNUNKNOWNOK1testb39a943d-a4da-408f-871a-d06e6210cd11down+unknownstatus not foundcluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294361,"replay_state":"idle"}2026-03-23 19:32:45' 2026-03-23T19:32:50.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:32:50.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:32:50.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update= 2026-03-23T19:32:50.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:32:50.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=down+unknown 2026-03-23T19:32:50.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo down+unknown 2026-03-23T19:32:50.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep down+unknown 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:down+unknown 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:32:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:32:50.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:32:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:32:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:32:50.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:50.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:50.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:50.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:32:50.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:50.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:50.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:32:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:50.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_image_meta cluster1 mirror test key1 value1 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local key=key1 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local value=value1 2026-03-23T19:32:50.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key1 2026-03-23T19:32:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test value1 = value1 2026-03-23T19:32:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_image_meta cluster1 mirror test key2 value2 2026-03-23T19:32:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:50.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:50.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:32:50.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local key=key2 2026-03-23T19:32:50.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local value=value2 2026-03-23T19:32:50.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key2 2026-03-23T19:32:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test value2 = value2 2026-03-23T19:32:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: stop mirror, add image, start mirror and test replay' 2026-03-23T19:32:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:32:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:32:50 TEST: stop mirror, add image, start mirror and test replay 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:32:50 TEST: stop mirror, add image, start mirror and test replay 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:32:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:50.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:32:50.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:32:50.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:32:50.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=65782 2026-03-23T19:32:50.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 65782 ']' 2026-03-23T19:32:50.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 65782 2026-03-23T19:32:50.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:50.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:50.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:50.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65782 '$2 == pid {print; exit 1}' 2026-03-23T19:32:50.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 65782 0.6 0.5 745524 57004 ? Ssl 19:32 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:50.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:50.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:51.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:51.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65782 '$2 == pid {print; exit 1}' 2026-03-23T19:32:51.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:32:51.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:51.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65782 '$2 == pid {print; exit 1}' 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:51.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:51.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:51.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:51.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:51.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:51.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:32:51.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:32:51.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:32:51.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:51.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:32:51.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:51.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:32:51.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:32:51.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:32:51.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=65826 2026-03-23T19:32:51.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 65826 ']' 2026-03-23T19:32:51.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 65826 2026-03-23T19:32:51.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:51.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:51.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:51.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65826 '$2 == pid {print; exit 1}' 2026-03-23T19:32:51.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 65826 0.3 0.4 671756 45928 ? Ssl 19:32 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:52.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:52.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65826 '$2 == pid {print; exit 1}' 2026-03-23T19:32:52.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:32:52.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:52.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65826 '$2 == pid {print; exit 1}' 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:52.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:52.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:52.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:52.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:52.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:52.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:52.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:52.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:32:52.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:52.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:52.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:32:52.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:32:52.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:32:52.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=65887 2026-03-23T19:32:52.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 65887 ']' 2026-03-23T19:32:52.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 65887 2026-03-23T19:32:52.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:52.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:52.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:52.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65887 '$2 == pid {print; exit 1}' 2026-03-23T19:32:52.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 65887 0.3 0.4 674828 47584 ? Ssl 19:32 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:52.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:52.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:53.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:53.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65887 '$2 == pid {print; exit 1}' 2026-03-23T19:32:53.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:32:53.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:53.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65887 '$2 == pid {print; exit 1}' 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:53.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:53.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:53.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:53.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:53.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:53.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:32:53.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:32:53.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:32:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:32:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:32:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:53.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:53.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:32:54.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:32:54.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:32:54.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=65957 2026-03-23T19:32:54.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 65957 ']' 2026-03-23T19:32:54.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 65957 2026-03-23T19:32:54.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:54.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:32:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:54.009 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65957 '$2 == pid {print; exit 1}' 2026-03-23T19:32:54.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 65957 0.2 0.4 671752 46564 ? Ssl 19:32 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:54.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:32:54.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:55.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:55.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65957 '$2 == pid {print; exit 1}' 2026-03-23T19:32:55.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:32:55.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:32:55.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=65957 '$2 == pid {print; exit 1}' 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:55.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:55.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:55.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:55.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:55.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:55.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:55.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:32:55.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:32:55.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image1=test1 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test1 snapshot 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test1 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:32:55.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test1 2026-03-23T19:32:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:32:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test1 snapshot 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:32:55.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test1 snapshot 2026-03-23T19:32:55.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:32:55.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test1 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test1': 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 11296ab27f84 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.11296ab27f84 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:32:55 2026 2026-03-23T19:32:55.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:32:55 2026 2026-03-23T19:32:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:32:55 2026 2026-03-23T19:32:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:32:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:32:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:32:55.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test1 100 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:32:55.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:32:55.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:32:55.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:32:55.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2083.33 bytes/sec: 8.1 MiB/s 2026-03-23T19:32:55.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:32:55.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:55.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:32:55.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:55.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:55.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:55.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:55.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:32:55.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:32:55.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:55.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:32:56.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:32:56.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:56.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:56.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:56.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:56.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:56.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:32:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-03-23T19:32:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:56.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:32:56.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:32:56.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:32:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:32:56.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-03-23T19:32:56.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-03-23T19:32:56.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:56.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:56.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:32:56.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:32:56.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:56.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:56.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:32:56.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-03-23T19:32:56.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-03-23T19:32:56.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:56.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:32:56.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:32:56.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:56.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:56.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:32:56.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-03-23T19:32:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-03-23T19:32:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:32:56.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:32:56.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:56.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:32:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:32:57.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:32:57.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:32:57.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:32:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:32:57.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:32:57.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:32:57.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:32:57.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:32:57.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:32:57.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:32:57.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:32:57.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:32:57.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test1 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test1 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:57.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test1 2026-03-23T19:32:58.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 10 2026-03-23T19:32:58.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test1 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test1.status 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test1.status 2026-03-23T19:32:58.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:58.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.status 2026-03-23T19:32:58.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test1.status 2026-03-23T19:32:58.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test1.status 2026-03-23T19:32:58.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test1 /tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.status 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.status 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test1 --format xml 2026-03-23T19:32:58.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:58.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:32:58.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=10 2026-03-23T19:32:58.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:32:58.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:58.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:32:58.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:58.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-03-23T19:32:58.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=9 2026-03-23T19:32:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 10 = 9 2026-03-23T19:32:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:32:58.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-03-23T19:32:58.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:58.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:58.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=9 2026-03-23T19:32:58.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 10 = 9 2026-03-23T19:32:58.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:32:58.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.status 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-03-23T19:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:32:59.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=10 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 10 = 10 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test1 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:32:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"} 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:00.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:57' 2026-03-23T19:33:00.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:00.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:00.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:00.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"} 2026-03-23T19:33:00.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:00.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:00.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:00.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:00.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:00.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:57' 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:32:57' 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:00.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:00.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:32:57' 2026-03-23T19:33:00.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:00.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:00.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:00.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294366,"remote_snapshot_timestamp":1774294366,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:32:59test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:32:57' 2026-03-23T19:33:00.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:00.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:00.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:57' 2026-03-23T19:33:00.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:00.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:00.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:00.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:00.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:00.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:00.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:00.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"} 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:57 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 9 .mirror.primary.b9361224-b07f-49fc-86ec-c1b3d2a8a822.23680472-084b-453a-ba00-4f98ed24d952 (peer_uuids:[]) 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 10 .mirror.primary.b9361224-b07f-49fc-86ec-c1b3d2a8a822.9830eb94-63d7-4821-a7cb-7c77a8ae82b6 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:33:01.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*down+unknown' 2026-03-23T19:33:01.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: down+unknown 2026-03-23T19:33:01.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:01.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: status not found 2026-03-23T19:33:01.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:01.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo down+unknown 2026-03-23T19:33:01.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:01.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: ' 2026-03-23T19:33:01.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:01.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update= 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=down+unknown 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local prev_last_update= 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:01.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='UNKNOWNUNKNOWNOK2testb39a943d-a4da-408f-871a-d06e6210cd11down+unknownstatus not foundcluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294366,"remote_snapshot_timestamp":1774294366,"replay_state":"idle"}2026-03-23 19:32:59test1b9361224-b07f-49fc-86ec-c1b3d2a8a822down+unknownstatus not foundcluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"}2026-03-23 19:32:57' 2026-03-23T19:33:01.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:01.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:01.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update= 2026-03-23T19:33:01.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:01.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=down+unknown 2026-03-23T19:33:01.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo down+unknown 2026-03-23T19:33:01.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep down+unknown 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:down+unknown 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test1 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:33:01.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:01.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test1.export 2026-03-23T19:33:01.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test1.export 2026-03-23T19:33:01.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:01.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.export 2026-03-23T19:33:01.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test1.export 2026-03-23T19:33:01.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test1.export 2026-03-23T19:33:01.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:01.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.export 2026-03-23T19:33:01.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.export 2026-03-23T19:33:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.export 2026-03-23T19:33:02.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:33:02.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.export 2026-03-23T19:33:02.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:33:02.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.export 2026-03-23T19:33:02.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test1.export 2026-03-23T19:33:02.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:02.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: test the first image is replaying after restart' 2026-03-23T19:33:02.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:33:02.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:33:02 TEST: test the first image is replaying after restart 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:33:02 TEST: test the first image is replaying after restart 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:33:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:33:02.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:33:02.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:33:02.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2777.77 bytes/sec: 11 MiB/s 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:02.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:03.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:03.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:03.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:03.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:03.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:03.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:03.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:03.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:03.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:03.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:03.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:03.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:03.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:03.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:03.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:03.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:03.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:33:05.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 11 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:33:05.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:05.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:33:05.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:05.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:33:05.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:33:05.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:33:05.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:33:05.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:33:05.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:33:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=11 2026-03-23T19:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:33:05.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:33:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:33:05.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=11 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 11 = 11 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:05.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:05.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294366,"remote_snapshot_timestamp":1774294366,"replay_state":"idle"} 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:32:59' 2026-03-23T19:33:06.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:06.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:06.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294366,"remote_snapshot_timestamp":1774294366,"replay_state":"idle"} 2026-03-23T19:33:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:06.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:06.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:06.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:06.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:06.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:06.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:59' 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:32:59' 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:32:59' 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:06.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:06.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294366,"remote_snapshot_timestamp":1774294366,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:32:59test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294375,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:32:57' 2026-03-23T19:33:06.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:06.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:06.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:32:59' 2026-03-23T19:33:06.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:06.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:33:06.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:06.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:33:06.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:33:06.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:33:06.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:33:06.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:33:06.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:33:06.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:33:07.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:33:07.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:33:07.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:33:07.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:33:07.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:33:07.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:07.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:33:07.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: stop/start/restart mirror via admin socket' 2026-03-23T19:33:07.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:33:07.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:33:07.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:33:07 TEST: stop/start/restart mirror via admin socket 2026-03-23T19:33:07.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:33:07 TEST: stop/start/restart mirror via admin socket 2026-03-23T19:33:07.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-03-23T19:33:07.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:07.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:07.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:07.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:07.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:07.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:07.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:07.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:07.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:07.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:07.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:07.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:07.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:07.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:07.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:07.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:07.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:07.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:07.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:07.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:07.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:07.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:07.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:07.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:07.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:07.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:08.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:08.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:08.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:08.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:08.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:08.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:08.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:08.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:08.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:08.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:08.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:08.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:08.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:08.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:08.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:08.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:08.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:08.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:08.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:08.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:08.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:08.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:09.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:09.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:09.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:09.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:09.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:09.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:09.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:09.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:33:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:10.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:10.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:07' 2026-03-23T19:33:10.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:10.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:10.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:10.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:10.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:10.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:10.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:10.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:10.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:10.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:10.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:10.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:07' 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:33:07' 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:07' 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:10.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:10.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:07test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:07' 2026-03-23T19:33:10.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:10.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:10.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:07' 2026-03-23T19:33:10.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:10.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:10.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:10.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:10.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:10.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:10.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:11.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:11.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:11.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:11.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:11.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:11.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:11.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:07' 2026-03-23T19:33:11.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:11.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:11.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:11.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:11.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:11.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:11.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:11.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:11.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:11.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:11.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:11.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:07' 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-03-23 19:33:07' 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:07' 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:11.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:11.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:07test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:07' 2026-03-23T19:33:11.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:11.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:11.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:07' 2026-03-23T19:33:11.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:11.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:11.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:11.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:11.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror start 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:11.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror start 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:11.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:11.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:11.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:11.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:11.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror start 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:11.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:11.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:11.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:11.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:11.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:11.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:11.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:11.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror start 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:11.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:11.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:11.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:11.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:11.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror start 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:11.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:11.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:12.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:12.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:13.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:13.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:13.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:13.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:13.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:13.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:13.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:13.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:13.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:13.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:13.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:13.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:13.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:13.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:13.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:13.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:13.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:13.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:13.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:13.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:13.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:13.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:13.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:13.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:13.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:14.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:14.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:14.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:14.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:14.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:14.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:14.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:14.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:15.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:15.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:11' 2026-03-23T19:33:15.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:15.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:15.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:15.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:15.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:15.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:15.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:11' 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:11' 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:11' 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:15.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:15.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:11test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:11' 2026-03-23T19:33:15.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:15.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:11' 2026-03-23T19:33:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:15.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:15.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:15.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:16.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:16.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:16.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:16.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:16.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:11' 2026-03-23T19:33:16.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:16.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:16.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:16.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:16.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:16.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:16.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:16.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:16.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:16.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:16.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:16.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:11' 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:33:11' 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:11' 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:16.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:11test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:11' 2026-03-23T19:33:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:17.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:11' 2026-03-23T19:33:17.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:17.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:17.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:17.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:17.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:17.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:17.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:17.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:17.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:17.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:17.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:17.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:17.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:17.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:17.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:17.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:17.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:17.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:17.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:17.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:17.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:17.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:17.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:17.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-03-23T19:33:17.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:17.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:17.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:17.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:17.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:17.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:17.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:18.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:18.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:18.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:18.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:18.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:18.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:18.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:18.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:18.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:18.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:18.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:18.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:18.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:18.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:18.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:19.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:19.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:19.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:19.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:19.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:19.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:19.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:19.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:19.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:19.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:19.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:19.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:19.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:19.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:19.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:19.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:20.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:20.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:17' 2026-03-23T19:33:20.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:20.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:20.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:20.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:20.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:20.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:20.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:20.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:20.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:20.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:20.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:20.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:17' 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:17' 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:17' 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:20.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:20.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:17test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:17' 2026-03-23T19:33:20.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:20.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:20.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:17' 2026-03-23T19:33:20.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:20.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:20.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:20.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:20.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:20.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:21.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:17' 2026-03-23T19:33:21.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:21.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:21.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:21.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:21.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:17' 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:33:17' 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:17' 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:21.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:17test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:17' 2026-03-23T19:33:21.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:21.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:21.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:17' 2026-03-23T19:33:21.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:21.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:21.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:21.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:21.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:21.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:21.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:21.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:21.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:21.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:21.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:21.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:21.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:21.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-03-23T19:33:21.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:21.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-03-23T19:33:21.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:21.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:21.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:21.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:21.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:22.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:22.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:22.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:22.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:22.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:22.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:22.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:22.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:22.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:22.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:22.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:22.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:23.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:23.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:23.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:23.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:23.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:23.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:23.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:23.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:23.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:23.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:23.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:23.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:23.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:24.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:24.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:24.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:24.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:24.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:24.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:24.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:24.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:24.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:24.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:24.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:24.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:24.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:24.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:24.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:24.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:24.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:24.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:25.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:25.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:25.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:21' 2026-03-23T19:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:25.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:25.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:25.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:25.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:25.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:25.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:25.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:25.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:25.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:25.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:21' 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:33:21' 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:21' 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:25.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:25.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:21test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:21' 2026-03-23T19:33:25.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:25.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:25.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:21' 2026-03-23T19:33:25.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:25.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:25.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:25.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:25.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:25.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:25.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:26.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:21' 2026-03-23T19:33:26.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:26.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:26.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:26.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:26.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:26.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:26.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:26.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:26.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:26.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:26.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:26.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:26.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:21' 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-03-23 19:33:21' 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:21' 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:26.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:26.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:21test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:21' 2026-03-23T19:33:26.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:26.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:26.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:21' 2026-03-23T19:33:26.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:26.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:26.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:26.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:26.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:26.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:26.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:26.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:26.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:26.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:26.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:26.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:26.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:26.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:26.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:26.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:26.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:26.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:26.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:26.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:26.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:26.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:26.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:26.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:26.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:26.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:26.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:27.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:27.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:27.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:27.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:27.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:27.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:27.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:27.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:27.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:27.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:27.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:27.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:27.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:27.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:27.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:27.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:27.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:27.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:27.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:27.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:27.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:27.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:27.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:27.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:27.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:27.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:27.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:28.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:28.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:28.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:28.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:28.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:28.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:28.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:28.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:28.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:28.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:28.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:28.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:28.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:28.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:29.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:29.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:26' 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:29.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:29.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:29.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:29.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:29.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:29.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:29.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:29.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:29.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:26' 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:26' 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:26' 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:29.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:29.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:26test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:26' 2026-03-23T19:33:29.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:29.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:29.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:26' 2026-03-23T19:33:29.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:29.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:29.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:29.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:29.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:29.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:30.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:30.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:30.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:30.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:30.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:30.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:30.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:30.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:30.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:30.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:26' 2026-03-23T19:33:31.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:31.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:31.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:31.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:31.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:31.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:31.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:31.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:31.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:31.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:31.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:31.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:31.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:26' 2026-03-23T19:33:31.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:33:26' 2026-03-23T19:33:31.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:26' 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:31.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:26test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:26' 2026-03-23T19:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:31.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:26' 2026-03-23T19:33:31.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:31.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:31.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:31.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:31.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:31.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:31.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:31.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:31.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:31.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:31.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:31.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:31.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:31.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:31.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:31.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:31.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:31.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:31.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:31.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:31.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:31.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:31.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:31.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:31.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:31.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:31.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:31.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:31.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:31.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:31.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:31.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:31.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:32.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:32.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:32.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:32.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:32.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:32.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:32.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:32.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:32.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:32.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:32.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:32.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:32.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:32.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:32.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:32.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:32.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:32.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:33.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:33.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:33.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:33.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:33.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:33.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:33.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:34.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:34.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:31' 2026-03-23T19:33:34.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:34.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:34.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:34.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:34.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:34.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:34.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:34.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:34.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:34.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:34.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:34.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:34.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:31' 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:33:31' 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:31' 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:34.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:34.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:31test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:31' 2026-03-23T19:33:34.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:34.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:34.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:31' 2026-03-23T19:33:34.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:34.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:34.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:34.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:34.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:34.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:34.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:35.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:31' 2026-03-23T19:33:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:35.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:35.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:35.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:35.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:35.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:35.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:35.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:35.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:35.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:35.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:35.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:31' 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-03-23 19:33:31' 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:31' 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:35.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:31test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:31' 2026-03-23T19:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:31' 2026-03-23T19:33:35.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:35.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:35.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:35.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:35.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:35.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:35.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:35.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:35.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-03-23T19:33:35.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:35.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror start mirror/test1 2026-03-23T19:33:35.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:35.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:35.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:35.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:35.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:35.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:35.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:36.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:36.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:36.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:36.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:36.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:36.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:36.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:36.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:36.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:36.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:36.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:36.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:36.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:36.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:37.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:37.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:37.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:37.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:37.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:37.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:38.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:38.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:35' 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:38.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:38.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:38.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:38.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:38.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:38.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:38.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:38.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:38.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:38.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:35' 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:35' 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:35' 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:38.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:38.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:35test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:31' 2026-03-23T19:33:38.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:38.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:38.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:35' 2026-03-23T19:33:38.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:38.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:38.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:38.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:38.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:38.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:38.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:38.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:38.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:38.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:38.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:38.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:38.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:38.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:38.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:38.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:38.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:38.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:38.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:38.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:38.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:38.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start mirror cluster2-DNE 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:38.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:39.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:39.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:39.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:39.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:39.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:39.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:39.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:39.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:39.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:39.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:39.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:39.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:39.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:38' 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:40.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:40.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:40.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:40.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:40.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:40.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:40.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:40.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:40.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:40.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:40.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:38' 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:33:38' 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:38' 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:40.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:40.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:35test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:38' 2026-03-23T19:33:40.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:40.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:40.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:38' 2026-03-23T19:33:40.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:40.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:40.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:40.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:40.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:40.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:40.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:40.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:40.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:40.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:40.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test 2026-03-23T19:33:40.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:40.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror restart mirror/test1 2026-03-23T19:33:40.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:40.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:40.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:40.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:41.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:41.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:41.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:41.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:41.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:41.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:41.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:41.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:41.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:41.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:41.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:41.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:41.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:41.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:41.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:41.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:41.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:41.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:41.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:41.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:42.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:40' 2026-03-23T19:33:42.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:42.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:42.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:42.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:42.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:42.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:42.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:42.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:42.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:42.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:42.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:42.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:40' 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:40' 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:40' 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:42.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:42.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:40test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:38' 2026-03-23T19:33:42.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:42.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:42.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:40' 2026-03-23T19:33:42.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:42.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:42.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:42.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:42.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:42.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:42.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:42.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:42.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:42.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:42.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:42.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:42.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:42.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:42.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:42.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:43.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:43.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:43.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:43.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:43.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:43.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:43.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:43.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:43.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:44.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:44.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:44.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:44.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:44.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:44.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:44.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:44.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:44.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:44.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:44.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:44.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:44.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:44.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:44.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:44.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:44.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:44.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:44.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:44.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:44.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:44.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:44.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:44.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:44.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:45.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:45.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:45.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:45.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:45.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:45.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:45.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:45.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:45.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:45.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:45.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:45.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:45.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:45.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:45.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:45.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:45.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:45.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:45.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:45.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:45.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:45.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:45.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:45.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:45.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:45.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:45.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:45.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:45.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:45.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:45.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:45.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:45.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:45.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:46.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:46.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:46.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:46.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:46.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:46.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:46.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:46.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:46.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:46.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:46.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:46.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:46.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:46.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:46.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:46.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:46.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:47.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:47.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:47.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:47.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:47.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:47.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:47.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:47.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:47.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:47.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:47.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:33:47.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:33:47.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:47.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:48.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:45' 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:48.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:48.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:48.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:48.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:48.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:48.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:45' 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:33:45' 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:45' 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:48.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:48.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:45test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:45' 2026-03-23T19:33:48.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:48.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:48.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:45' 2026-03-23T19:33:48.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:48.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:48.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:48.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:48.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:49.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:49.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:49.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: stopped 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:45' 2026-03-23T19:33:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:33:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:33:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:49.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: stopped 2026-03-23T19:33:49.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:49.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:49.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:49.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:49.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:49.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:45' 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-03-23 19:33:45' 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:45' 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:49.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+stoppedstopped44094414mirror.1vm08.local2026-03-23 19:33:45test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+stoppedstopped44024407mirror.0vm08.local2026-03-23 19:33:45' 2026-03-23T19:33:49.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:49.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:49.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:45' 2026-03-23T19:33:49.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:49.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:33:49.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:33:49.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:33:49.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:49.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:49.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:49.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:49.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:49.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:49.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:50.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:50.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:50.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:50.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:50.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:50.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:50.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:50.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:50.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:50.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:50.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:50.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:50.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:50.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:51.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:51.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:51.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:51.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:51.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:51.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:33:51.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:no valid command found; 1 closest matches: 2026-03-23T19:33:51.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status mirror/test1 2026-03-23T19:33:51.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:51.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:51.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:51.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:51.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:51.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:51.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:33:51.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:51.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:51.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:51.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:51.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:51.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:51.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:51.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:51.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:52.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:52.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:52.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:52.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:52.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:52.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-03-23T19:33:52.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:52.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:52.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:53.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:53.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:50' 2026-03-23T19:33:53.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:53.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:53.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:50' 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:33:50' 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:50' 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:53.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:53.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:50test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:50' 2026-03-23T19:33:53.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:53.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:33:53.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:50' 2026-03-23T19:33:53.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:33:53.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:53.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:53.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:54.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:54.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test1: 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b9361224-b07f-49fc-86ec-c1b3d2a8a822 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:50' 2026-03-23T19:33:54.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:54.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:54.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:54.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"} 2026-03-23T19:33:54.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:54.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:54.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:54.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:54.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:54.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:33:54.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:54.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:50' 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-03-23 19:33:50' 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:50' 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:54.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:54.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:50test1b9361224-b07f-49fc-86ec-c1b3d2a8a822up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294378,"remote_snapshot_timestamp":1774294378,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:33:50' 2026-03-23T19:33:54.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:54.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-03-23T19:33:54.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:50' 2026-03-23T19:33:54.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-03-23T19:33:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ flush cluster1 2026-03-23T19:33:54.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool= 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image= 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'cmd=rbd mirror flush' 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror flush 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:54.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:54.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:54.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror flush 2026-03-23T19:33:54.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:54.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:54.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:54.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:54.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush 2026-03-23T19:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ all_admin_daemons cluster1 rbd mirror status 2026-03-23T19:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror status 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:54.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:54.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:54.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:{ 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool_replayers": [ 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: 04da4782-4604-444b-8d54-7d24cb321a84 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4407", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4407", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": true, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instances": [ 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4407", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4414", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4428", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4432" 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ], 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.68023.cluster1.94774260013440.asok", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68023.cluster2-DNE.94774271244672.asok", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [ 2026-03-23T19:33:54.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "name": "mirror/test1", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter_status": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "delete_images_queue": [], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "failed_deletes_queue": [] 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter_status": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "delete_images_queue": [], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "failed_deletes_queue": [] 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns2", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns2", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter_status": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "delete_images_queue": [], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "failed_deletes_queue": [] 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror_parent", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4419", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4419", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": true, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instances": [ 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4419", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4429", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4434", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "4437" 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ], 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.68023.cluster1.94774263617920.asok", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68023.cluster2-DNE.94774266468736.asok", 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [], 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter": { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter_status": { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "delete_images_queue": [], 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "failed_deletes_queue": [] 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [], 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter": { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_deleter_status": { 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "delete_images_queue": [], 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "failed_deletes_queue": [] 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:} 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror status 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:54.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:54.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:54.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:54.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:54.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:54.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:{ 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool_replayers": [ 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: 04da4782-4604-444b-8d54-7d24cb321a84 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4414", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4407", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.68067.cluster1.94098461866368.asok", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68067.cluster2-DNE.94098475096448.asok", 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [ 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "name": "mirror/test", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns2", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns2", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror_parent", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4429", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4419", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.68067.cluster1.94098465626496.asok", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68067.cluster2-DNE.94098468395392.asok", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:} 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:2 rbd mirror status 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:54.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:33:54.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:33:54.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:54.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:54.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:33:54.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:{ 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool_replayers": [ 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: 04da4782-4604-444b-8d54-7d24cb321a84 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror", 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4428", 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4407", 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.68119.cluster1.94717902543232.asok", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68119.cluster2-DNE.94717916027264.asok", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns2", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns2", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror_parent", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4434", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4419", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.68119.cluster1.94717906540928.asok", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68119.cluster2-DNE.94717909334400.asok", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:} 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:3 rbd mirror status 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:33:54.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:33:54.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:33:54.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:54.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:54.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:33:54.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:{ 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool_replayers": [ 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: 04da4782-4604-444b-8d54-7d24cb321a84 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror", 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4432", 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4407", 2026-03-23T19:33:54.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.68205.cluster1.94221557275008.asok", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68205.cluster2-DNE.94221569735040.asok", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns2", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns2", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "peer": "uuid: a11e0708-b1cb-4d9c-bb97-5f93ef21bc27 cluster: cluster2-DNE client: client.mirror", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "pool": "mirror_parent", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "instance_id": "4437", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "stopped (manual)", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader_instance_id": "4419", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "leader": false, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.68205.cluster1.94221560879488.asok", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.68205.cluster2-DNE.94221562665344.asok", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "sync_throttler": { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 5, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "deletion_throttler": { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "max_parallel_requests": 1, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "running_requests": 0, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "waiting_requests": 0 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "namespace_replayers": [ 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: }, 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: { 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "local_namespace": "ns1", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "remote_namespace": "ns1", 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "image_replayers": [] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: } 2026-03-23T19:33:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: ] 2026-03-23T19:33:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:} 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test1 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:33:54.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test1 2026-03-23T19:33:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test1 2026-03-23T19:33:54.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test1 2026-03-23T19:33:54.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test1 2026-03-23T19:33:55.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:33:55.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:55.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: test image rename' 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:33:55 TEST: test image rename 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:33:55 TEST: test image rename 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ new_name=test_RENAMED 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_image cluster2 mirror test test_RENAMED 2026-03-23T19:33:55.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:55.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:55.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:33:55.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_name=test_RENAMED 2026-03-23T19:33:55.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rename mirror/test mirror/test_RENAMED 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_RENAMED 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_RENAMED 2026-03-23T19:33:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 12 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:58.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:33:58.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:58.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:33:58.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:58.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:58.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:33:58.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:58.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:33:58.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:58.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:58.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:58.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:58.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:58.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-03-23T19:33:58.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:33:58.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:58.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:33:58.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:58.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:58.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:58.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:58.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:58.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:33:58.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:33:58.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:33:58.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:58.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:58.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:58.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:58.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:59.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-03-23T19:33:59.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_RENAMED: 2026-03-23T19:33:59.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:33:59.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:33:59.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:59.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:33:59.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:55' 2026-03-23T19:33:59.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:33:59.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:33:59.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:33:59.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"} 2026-03-23T19:33:59.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:33:59.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:59.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:33:59.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:59.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:33:59.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:33:59.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:33:59.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:55' 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-03-23 19:33:55' 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:55' 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:33:59.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:33:59.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:33:59.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_RENAMEDb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294384,"remote_snapshot_timestamp":1774294384,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:55' 2026-03-23T19:33:59.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:33:59.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-03-23T19:33:59.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:55' 2026-03-23T19:33:59.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-03-23T19:33:59.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:33:59.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:33:59.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:33:59.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:33:59.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:59.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:59.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:59.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:59.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:59.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-03-23T19:33:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:33:59.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:59.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:59.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:33:59.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:{ 2026-03-23T19:33:59.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "name": "mirror/test_RENAMED", 2026-03-23T19:33:59.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:33:59.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:} 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test_RENAMED 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:59.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:33:59.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:33:59.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:33:59.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:59.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test_RENAMED 2026-03-23T19:33:59.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:33:59.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:59.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:33:59.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:33:59.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:33:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:33:59.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test_RENAMED 2026-03-23T19:33:59.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-03-23T19:33:59.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror restart : restart rbd mirror 2026-03-23T19:33:59.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test_RENAMED 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:33:59.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:33:59.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:33:59.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:33:59.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test_RENAMED 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:33:59.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:00.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:00.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:00.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:00.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:00.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:00.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:01.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:01.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:01.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:01.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:34:01.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-03-23T19:34:01.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:01.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:01.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:34:01.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:34:01.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:01.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-03-23T19:34:01.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:01.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:01.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:01.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:01.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:01.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:01.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:02.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-03-23T19:34:02.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_RENAMED: 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b39a943d-a4da-408f-871a-d06e6210cd11 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294436,"remote_snapshot_timestamp":1774294436,"replay_state":"idle"} 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:33:59' 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:34:02.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:02.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294436,"remote_snapshot_timestamp":1774294436,"replay_state":"idle"} 2026-03-23T19:34:02.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:02.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:02.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:02.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:02.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:34:02.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:02.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:59' 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-03-23 19:33:59' 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:33:59' 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:02.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_RENAMEDb39a943d-a4da-408f-871a-d06e6210cd11up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294436,"remote_snapshot_timestamp":1774294436,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:33:59' 2026-03-23T19:34:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-03-23T19:34:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:33:59' 2026-03-23T19:34:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-03-23T19:34:02.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:34:02.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:02.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_image cluster2 mirror test_RENAMED test 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_RENAMED 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_name=test 2026-03-23T19:34:02.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rename mirror/test_RENAMED mirror/test 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:34:04.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 13 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:04.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:05.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:05.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:05.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:05.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:05.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:05.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:05.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:05.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:05.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:05.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:05.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:05.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:05.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:34:05.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:34:05.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:05.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:05.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:05.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:34:05.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:05.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:05.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:05.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:05.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:05.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:05.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:05.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:05.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: test trash move restore' 2026-03-23T19:34:05.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:34:05.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:34:05 TEST: test trash move restore 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:34:05 TEST: test trash move restore 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster2 mirror test 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:05.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:34:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:34:05.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=11092a1e7879 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ trash_move cluster2 mirror test 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:05.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 trash move mirror/test 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:06.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:34:06.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:34:06.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:34:06.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:34:06.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:34:06.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:34:06.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:34:06.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:34:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ trash_restore cluster2 mirror test 11092a1e7879 snapshot 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=11092a1e7879 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:34:06.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 trash restore mirror/11092a1e7879 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:34:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-03-23T19:34:07.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:34:07.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 11092a1e7879 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.11092a1e7879 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:32:40 2026 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: a80d3a3b-f05e-4973-aa6b-e098de587c21 2026-03-23T19:34:07.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:07.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:08.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:08.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:08.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:08.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:08.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:08.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:08.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:08.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:08.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:08.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:08.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:08.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:08.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:08.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:08.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:34:08.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:34:08.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:34:08.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:08.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:08.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:08.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:34:08.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:08.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:08.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:34:08.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:34:08.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:34:08.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:08.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:08.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:08.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:34:08.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:08.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:08.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:34:08.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:34:08.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:08.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:08.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:08.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:34:08.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:08.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:08.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:08.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:08.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:34:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:09.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:09.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:09.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:09.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:09.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:09.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:09.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:09.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:09.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:09.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed (with rbd-mirror on one cluster)' 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:34:09 TEST: check if removed 'images'\''' OMAP are removed '(with' rbd-mirror on one 'cluster)' 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:34:09 TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster) 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:09.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-03-23T19:34:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-03-23T19:34:10.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=status_global 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:10.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:10.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=status_global 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:34:10.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c status_global 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_ 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:10.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_ 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:10.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:34:10.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_ 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirror_leader 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_map 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:10.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:10.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:34:10.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 0 2026-03-23T19:34:10.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=1 2026-03-23T19:34:10.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:10.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 -eq 0 ']' 2026-03-23T19:34:10.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:10.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:11.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:34:11.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:34:11.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 0 2026-03-23T19:34:11.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=1 2026-03-23T19:34:11.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:11.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 -eq 0 ']' 2026-03-23T19:34:11.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:11.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:13.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=status_global 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:13.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=status_global 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c status_global 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:13.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_ 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_ 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_ 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:34:13.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirror_leader 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_map 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:34:13.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:13.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:34:13.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:34:13.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:error getting omap key set mirror/rbd_mirror_leader: (2) No such file or directory 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:13.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:34:13.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-03-23T19:34:14.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:34:14.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 118eeabff75e 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.118eeabff75e 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:34:13 2026 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:34:13 2026 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:34:13 2026 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:14.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:14.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:15.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:15.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:15.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:15.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:15.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:15.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:15.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:15.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:15.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:15.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:15.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:15.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:15.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:34:15.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:34:15.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:15.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:15.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:34:15.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-03-23T19:34:15.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:15.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:15.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:34:15.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:34:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:15.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:15.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:34:15.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-03-23T19:34:15.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:15.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:15.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:34:15.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:34:15.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:34:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:34:15.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-03-23T19:34:15.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-03-23T19:34:15.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:34:15.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:34:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:34:16.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:16.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:16.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:16.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:16.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:16.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:16.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:16.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:16.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:16.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:16.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:16.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:34:16.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:34:16.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:34:16.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:34:16.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 3225.8 bytes/sec: 13 MiB/s 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:16.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:34:17.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 16 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:17.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:34:17.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:34:17.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:34:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:34:17.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:17.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:34:17.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:17.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:34:17.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=16 2026-03-23T19:34:17.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:34:17.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:34:17.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:34:17.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:17.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:34:17.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=15 2026-03-23T19:34:17.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 16 = 15 2026-03-23T19:34:17.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:34:17.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:34:17.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:17.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:34:17.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=15 2026-03-23T19:34:17.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 16 = 15 2026-03-23T19:34:17.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:34:17.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:34:18.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:18.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=16 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 16 = 16 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:18.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:19.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:19.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:19.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:19.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294454,"replay_state":"idle"} 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:16' 2026-03-23T19:34:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:34:19.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:34:19.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:19.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294454,"replay_state":"idle"} 2026-03-23T19:34:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:19.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:19.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:19.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:16' 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:34:16' 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:16' 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:19.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294454,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:34:16' 2026-03-23T19:34:19.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:19.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:19.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:16' 2026-03-23T19:34:19.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:19.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:34:19.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:19.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: failover and failback' 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:34:19.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:34:19.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:34:19 TEST: failover and failback 2026-03-23T19:34:19.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:34:19 TEST: failover and failback 2026-03-23T19:34:19.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster2 2026-03-23T19:34:19.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:19.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster2:1 ']' 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster2:2 ']' 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:19.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster2:3 ']' 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:34:19.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:34:19.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:19.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:19.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:19.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:34:21.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:34:21.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:21.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:22.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:22.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:22.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:22.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:22.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:22.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:22.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:22.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:22.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:22.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:34:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:23.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:23.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:34:23.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:22 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:21' 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:23.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:23.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:34:23.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:23.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:23.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:23.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:23.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:23.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:23.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:23.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:22' 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:34:22' 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:22' 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:23.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:23.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:34:22cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:34:21' 2026-03-23T19:34:23.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:23.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:23.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:22' 2026-03-23T19:34:23.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:23.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:23.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:24.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:34:24.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:24.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:24.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:24.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:21 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:22' 2026-03-23T19:34:24.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:24.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:24.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:24.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:34:24.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:24.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:24.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:24.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:24.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:24.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:34:24.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:24.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:26.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:24 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:25' 2026-03-23T19:34:26.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:26.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:26.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:26.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:34:26.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:26.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:26.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:26.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:26.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:26.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:26.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:26.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:24' 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:34:24' 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:24' 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:26.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:34:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:34:25' 2026-03-23T19:34:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:26.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:24' 2026-03-23T19:34:26.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:26.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:34:26.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:34:28.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:28.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:29.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:29.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:29.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:29.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:29.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:29.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:29.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:29.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:29.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:29.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:29.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:29.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:29.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:29.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:29.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:29.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:34:29.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:29.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:31.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:31.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:31.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:31.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:31.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:31.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:31.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:31.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:31.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:31.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:31.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:31.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:31.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:31.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:34:31.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:34:31.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:34:31.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:34:31.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2127.65 bytes/sec: 8.3 MiB/s 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:31.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:34:33.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 19 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:34:33.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:34:33.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:33.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:33.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:34:33.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:34:33.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:34:33.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:34:33.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:33.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:34:33.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=19 2026-03-23T19:34:33.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:34:33.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:34:33.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:34:33.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:33.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:33.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:33.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:33.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:34:33.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:34:33.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:34:33.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=19 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 19 = 19 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:33.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:34.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:34.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:29 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294460,"remote_snapshot_timestamp":1774294467,"replay_state":"idle"} 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:30 2026-03-23T19:34:34.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:34:34.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 18 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.101b24e2-3be9-464b-96ad-ce93b7758e62 (peer_uuids:[]) 2026-03-23T19:34:34.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 19 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.9e96443f-587e-4aec-b09e-8c29e00bbc9c (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:34:34.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:34:34.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:34:34.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:34.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:34:34.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:34.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:34:34.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:34.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:34:34.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:34.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:34.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:34.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:29' 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:34:29' 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:29' 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:34.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:34.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:34:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294460,"remote_snapshot_timestamp":1774294467,"replay_state":"idle"}2026-03-23 19:34:30' 2026-03-23T19:34:34.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:34.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:34.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:29' 2026-03-23T19:34:34.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:34.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:34:34.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:34:34.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:34:34.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:34.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294460,"remote_snapshot_timestamp":1774294467,"replay_state":"idle"} 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:30 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:34' 2026-03-23T19:34:35.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:34:35.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:34:35.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:35.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294460,"remote_snapshot_timestamp":1774294467,"replay_state":"idle"} 2026-03-23T19:34:35.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:35.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:35.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:35.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:35.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:35.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:35.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:35.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:30' 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:34:30' 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:30' 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:35.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294460,"remote_snapshot_timestamp":1774294467,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:34:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:34:34' 2026-03-23T19:34:35.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:35.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:35.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:30' 2026-03-23T19:34:35.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:34:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:34:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:34:35.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:34:35.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:35.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:35.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:34:35.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:35.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:34:35.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:34:35.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:34:35.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:34:35.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:34:35.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:34:35.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:34:35.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:34:35.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:34:35.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:34:35.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:34:35.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:35.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:34:37.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:38.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:38.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:38.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:38.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:38.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:38.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:38.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:38.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:38.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:38.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:34:38.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:38.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:34:38.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:38.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:38.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:38 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:34' 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:39.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:39.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:34:39.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:39.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:39.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:39.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:39.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:39.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:39.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:39.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:38' 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:34:38' 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:38' 2026-03-23T19:34:39.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:39.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:39.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:39.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:34:38cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:34:34' 2026-03-23T19:34:39.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:39.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:39.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:38' 2026-03-23T19:34:39.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:39.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:39.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:39.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:39.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:39.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:40.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:39 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:40' 2026-03-23T19:34:40.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:40.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:40.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:40.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:34:40.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:40.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:40.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:40.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:40.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:39' 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:34:39' 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:39' 2026-03-23T19:34:40.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:34:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:34:40' 2026-03-23T19:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:39' 2026-03-23T19:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:40.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:34:40.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:34:41.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:41.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:42.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:42.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:34:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:42.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:42.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:42.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:34:42.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:42.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:42.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:42.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:42.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:42.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:42.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:34:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:42.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:44.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:44.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:34:44.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:44.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:44.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:34:44.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:44.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:44.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:44.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:44.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:44.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:44.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:44.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:44.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:34:44.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:44.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:48.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:34:48.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:48.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:48.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:34:48.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:48.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:48.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:48.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:48.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:48.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:34:48.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:34:48.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:34:52.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:52.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:53.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:53.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:34:53.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:53.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:53.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:34:53.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:53.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:53.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:34:53.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:53.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:53.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:53.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:53.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:53.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:53.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:54.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:50 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:50' 2026-03-23T19:34:54.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:54.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:54.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:54.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:54.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:54.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:50' 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:34:50' 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:50' 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:54.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:34:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:34:50' 2026-03-23T19:34:54.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:54.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:50' 2026-03-23T19:34:54.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:54.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:54.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:54.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:55.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:55.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:54 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:55' 2026-03-23T19:34:55.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:34:55.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:34:55.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:34:55.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:34:55.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:34:55.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:55.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:34:55.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:55.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:34:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:34:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:34:55.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:54' 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:34:54' 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:54' 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:34:55.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:34:55.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:34:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:34:55' 2026-03-23T19:34:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:34:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:34:55.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:54' 2026-03-23T19:34:55.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:34:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:34:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:34:55.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:34:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:34:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:55.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:34:55.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:34:57.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:34:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:34:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:57.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:57.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:34:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:34:58.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:58.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:58.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:34:58.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:34:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:34:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:34:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:34:58.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:34:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:34:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:34:58.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:00.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:00.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:00.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:00.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:00.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:00.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:35:02.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 24 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:02.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:02.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:35:02.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:35:02.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:02.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:02.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:35:02.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:35:02.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:35:02.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:02.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:35:02.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=24 2026-03-23T19:35:02.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:35:02.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:35:02.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:35:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:02.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:02.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:35:02.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:02.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=24 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 24 = 24 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:02.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:02.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:03.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:35:03.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:03.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:03.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:03.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:03.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:03.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294496,"replay_state":"idle"} 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:00 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:59' 2026-03-23T19:35:03.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:35:03.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:35:03.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:03.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294496,"replay_state":"idle"} 2026-03-23T19:35:03.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:03.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:03.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:03.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:03.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:03.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:03.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:03.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:00' 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:35:00' 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:00' 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:03.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:03.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294496,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:35:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:34:59' 2026-03-23T19:35:03.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:03.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:03.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:00' 2026-03-23T19:35:03.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:03.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:35:03.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:03.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:03.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:04.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:04.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:04.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:04.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:34:59 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294496,"replay_state":"idle"} 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:00 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 20 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f3aa4eaa-0c41-44a7-a89b-91a55577815d (demoted peer_uuids:[]) 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 23 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.48054aa7-e42d-40e9-a9da-80aabd7e5ddb (peer_uuids:[]) 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 24 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.403fa49e-d30f-43fd-93e5-dcf1606d8bff (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:35:04.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:35:04.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:04.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:35:04.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:04.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:04.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:04.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:04.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:04.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:59' 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:34:59' 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:34:59' 2026-03-23T19:35:04.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:04.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:04.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:04.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:34:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294496,"replay_state":"idle"}2026-03-23 19:35:00' 2026-03-23T19:35:04.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:04.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:04.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:34:59' 2026-03-23T19:35:04.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:04.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:35:04.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:04.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:35:04.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:35:04.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:04.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:04.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:35:04.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:35:04.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:04.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:04.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:04.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:04.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:04.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:04.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:04.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:04.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:35:08.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:08.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:09.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:09.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:09.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:09.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:09.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:09.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:09.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:09.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:09.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:10.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:10.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:10.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:10.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:10 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:09' 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:10.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:10.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:10.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:10.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:10.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:10.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:10.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:10.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:10' 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:35:10' 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:10' 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:10.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:10.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:35:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:35:09' 2026-03-23T19:35:10.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:10.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:10.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:10' 2026-03-23T19:35:10.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:10.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:10.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:10.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:10.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:10.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:10.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:10.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:10.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:11.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:09 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:10' 2026-03-23T19:35:11.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:11.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:11.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:11.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:11.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:11.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:11.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:09' 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:35:09' 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:09' 2026-03-23T19:35:11.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:11.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:11.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:35:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:35:10' 2026-03-23T19:35:11.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:11.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:11.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:09' 2026-03-23T19:35:11.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:11.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:35:11.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:35:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:12.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:13.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:13.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:13.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:13.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:13.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:13.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:13.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:13.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:13.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:13.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:13.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:13.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:13.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:13.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:35:13.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:13.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:15.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:15.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:15.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:15.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:15.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:15.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:15.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:15.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:15.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:15.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:15.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:15.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:15.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:15.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster1 mirror test 100 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:35:15.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:35:15.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:35:15.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:35:15.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 3124.99 bytes/sec: 12 MiB/s 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:15.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:35:19.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 27 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:19.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:19.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:35:19.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:35:19.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:19.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:19.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:35:19.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:19.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:35:19.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=27 2026-03-23T19:35:19.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:35:19.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:35:19.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:35:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:35:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:19.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:35:19.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=27 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 27 = 27 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:19.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:20.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:20.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:20.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:20.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:20 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":95027.2,"bytes_per_snapshot":237568.0,"last_snapshot_bytes":475136,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294516,"remote_snapshot_timestamp":1774294516,"replay_state":"idle"} 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:19 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 27 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.5673c775-5746-4692-8efe-90445d01810d (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:35:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:35:20.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:35:20.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:35:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:20.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:20.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:20.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:20.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:20.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:20' 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:35:20' 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:20' 2026-03-23T19:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:20.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:20.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:20.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:35:20cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":95027.2,"bytes_per_snapshot":237568.0,"last_snapshot_bytes":475136,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294516,"remote_snapshot_timestamp":1774294516,"replay_state":"idle"}2026-03-23 19:35:19' 2026-03-23T19:35:20.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:20.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:20.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:20' 2026-03-23T19:35:20.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:20.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:35:20.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:20.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:35:20.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:20.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror test 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:20.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:21.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":95027.2,"bytes_per_snapshot":237568.0,"last_snapshot_bytes":475136,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294516,"remote_snapshot_timestamp":1774294516,"replay_state":"idle"} 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:19 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:20' 2026-03-23T19:35:21.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:35:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:35:21.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":95027.2,"bytes_per_snapshot":237568.0,"last_snapshot_bytes":475136,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294516,"remote_snapshot_timestamp":1774294516,"replay_state":"idle"} 2026-03-23T19:35:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:21.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:21.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:21.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:21.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:21.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:19' 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:35:19' 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:19' 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:21.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":95027.2,"bytes_per_snapshot":237568.0,"last_snapshot_bytes":475136,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294516,"remote_snapshot_timestamp":1774294516,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:35:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:35:20' 2026-03-23T19:35:21.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:21.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:19' 2026-03-23T19:35:21.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:21.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:35:21.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:21.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:21.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:21.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:35:21.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:35:21.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:21.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:21.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:35:21.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:35:21.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:21.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:21.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:21.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:21.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:21.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:21.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:21.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:21.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:21.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:35:22.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:35:22.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:22.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:23.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:23.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:23.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:23.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:23.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:23.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:23.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:23.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:23.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:23.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:23.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:23.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:23.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:23.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:23.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:23.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:23.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:23.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:23.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:35:23.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:23.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:25.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:25.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:25.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:25.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:25.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:25.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:25.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:25.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:25.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:25.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:25.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:25.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:25.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:26.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:26.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:25 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:24' 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:26.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:26.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:26.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:26.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:26.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:26.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:26.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:26.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:26.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:25' 2026-03-23T19:35:26.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:35:25' 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:25' 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:26.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:35:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:35:24' 2026-03-23T19:35:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:26.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:26.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:25' 2026-03-23T19:35:26.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:26.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:26.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:26.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:26.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:27.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:24 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:25' 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:27.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:27.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:27.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:27.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:27.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:27.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:27.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:27.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:27.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:27.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:27.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:24' 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:35:24' 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:24' 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:27.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:35:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:35:25' 2026-03-23T19:35:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:27.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:24' 2026-03-23T19:35:27.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:35:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:35:28.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:28.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:29.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:29.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:29.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:29.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:29.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:29.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:29.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:29.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:29.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:29.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:29.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:29.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:29.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:29.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:29.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:35:29.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:29.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:31.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:31.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:31.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:31.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:31.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:31.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:31.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:31.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:31.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:31.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:31.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:31.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:31.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:31.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:31.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:31.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test 100 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:35:31.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:35:31.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:35:31.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:35:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:35:31.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 3030.3 bytes/sec: 12 MiB/s 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:31.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:35:35.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 30 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:35:35.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:35.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:35.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:35:35.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:35:35.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:35:35.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:35.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:35:35.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=30 2026-03-23T19:35:35.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:35:35.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:35:35.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:35:35.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:35:35.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:35.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:35:35.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=30 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 30 = 30 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:35.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:36.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:36.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:36.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:36.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294528,"replay_state":"idle"} 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:30 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:34' 2026-03-23T19:35:37.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:35:37.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:35:37.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:37.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294528,"replay_state":"idle"} 2026-03-23T19:35:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:37.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:37.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:37.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:37.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:30' 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:35:30' 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:30' 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:37.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294528,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:35:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:35:34' 2026-03-23T19:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:37.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:37.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:30' 2026-03-23T19:35:37.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:37.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:35:37.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:37.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:37.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:37.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:38.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:34 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294528,"replay_state":"idle"} 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:30 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 30 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.1bab8d4f-5998-44eb-8cc7-aebff00b3cbf (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:35:38.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:35:38.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:35:38.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:38.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:35:38.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:38.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:38.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:38.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:38.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:38.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:38.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:34' 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:35:34' 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:34' 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:38.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:38.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:35:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294528,"replay_state":"idle"}2026-03-23 19:35:30' 2026-03-23T19:35:38.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:38.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:38.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:34' 2026-03-23T19:35:38.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:38.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:35:38.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:38.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:35:38.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:35:38.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:38.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:38.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:35:38.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:35:38.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:38.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:38.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:38.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:35:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:38.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:38.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:35:38.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:38.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:35:38.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:38.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: failover / failback loop' 2026-03-23T19:35:38.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:35:38.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:35:38.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:35:38 TEST: failover / failback loop 2026-03-23T19:35:38.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:35:38 TEST: failover / failback loop 2026-03-23T19:35:38.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 1 20 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:35:38.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:35:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:38.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:39.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:39.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:39.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:39.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:39.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:39.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:39.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:39.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:39.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:40.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:40.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:40.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:40.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:40.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:40.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:40.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:35:40.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:40.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:42.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:42.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:42.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:35:42.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:35:42.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:42.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:35:42.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:42.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:42.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:35:42.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:42.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:42.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:43.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:40 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:39' 2026-03-23T19:35:43.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:43.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:43.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:43.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:43.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:43.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:43.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:43.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:43.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:43.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:43.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:43.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:40' 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:35:40' 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:40' 2026-03-23T19:35:43.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:43.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:43.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:43.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:35:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:35:39' 2026-03-23T19:35:43.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:43.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:43.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:40' 2026-03-23T19:35:43.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:43.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:43.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:43.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:43.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:44.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:44.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:39 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:40' 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:44.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:44.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:44.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:44.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:44.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:44.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:44.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:44.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:44.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:44.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:39' 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:35:39' 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:39' 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:44.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:44.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:35:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:35:40' 2026-03-23T19:35:44.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:44.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:44.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:39' 2026-03-23T19:35:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:35:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:35:45.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:45.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:46.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:46.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:46.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:46.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:46.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:46.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:46.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:46.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:46.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:46.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:46.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:46.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:35:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:35:50.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 33 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:35:50.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:50.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:35:50.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:35:50.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:50.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:50.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:35:50.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:35:50.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:50.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:35:50.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:35:50.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:50.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:35:50.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=33 2026-03-23T19:35:50.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:35:50.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:35:50.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:35:50.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:35:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=33 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 33 = 33 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:50.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:51.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:51.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:50 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294547,"remote_snapshot_timestamp":1774294547,"replay_state":"idle"} 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:49 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 33 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.cf724e9d-8353-496c-917f-4f485e4a837f (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:35:51.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:51.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:35:51.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:51.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:51.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:51.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:51.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:51.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:51.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:51.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:50' 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:35:50' 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:50' 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:51.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:35:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294547,"remote_snapshot_timestamp":1774294547,"replay_state":"idle"}2026-03-23 19:35:49' 2026-03-23T19:35:51.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:51.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:50' 2026-03-23T19:35:51.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:51.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:35:51.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:35:51.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:35:51.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:35:51.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:51.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:52.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294547,"remote_snapshot_timestamp":1774294547,"replay_state":"idle"} 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:49 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:50' 2026-03-23T19:35:52.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:35:52.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:35:52.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:52.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294547,"remote_snapshot_timestamp":1774294547,"replay_state":"idle"} 2026-03-23T19:35:52.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:52.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:52.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:52.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:52.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:52.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:52.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:52.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:49' 2026-03-23T19:35:52.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:35:49' 2026-03-23T19:35:52.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:52.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:49' 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:52.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:52.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294547,"remote_snapshot_timestamp":1774294547,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:35:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:35:50' 2026-03-23T19:35:52.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:52.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:52.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:49' 2026-03-23T19:35:52.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:52.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:35:52.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:35:52.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:35:52.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:52.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:35:52.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:35:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:35:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:52.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:52.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:53.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:53.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:53.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:53.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:53.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:53.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:53.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:53.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:53.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:53.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:53.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:35:53.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:35:53.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:53.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:35:53.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:35:53.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:35:53.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:53.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:55.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:35:55.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:35:55.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:35:55.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:35:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:35:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:35:55.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:35:55.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:35:55.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:35:55.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:55.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:56.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:55 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:54' 2026-03-23T19:35:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:56.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:56.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:56.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:56.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:56.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:56.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:56.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:56.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:56.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:56.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:56.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:55' 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:35:55' 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:55' 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:56.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:56.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:35:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:35:54' 2026-03-23T19:35:56.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:56.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:56.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:55' 2026-03-23T19:35:56.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:56.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:56.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:56.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:56.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:56.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:56.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:56.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:57.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:54 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:35:55' 2026-03-23T19:35:57.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:35:57.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:35:57.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:35:57.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:35:57.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:35:57.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:57.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:35:57.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:57.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:35:57.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:35:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:35:57.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:54' 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:35:54' 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:35:54' 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:35:57.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:35:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:35:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:35:55' 2026-03-23T19:35:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:35:58.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:35:58.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:35:54' 2026-03-23T19:35:58.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:35:58.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:35:58.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:35:58.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:35:58.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:35:59.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:35:59.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:35:59.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:35:59.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:35:59.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:35:59.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:00.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:00.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:00.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:00.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:00.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:00.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:00.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:00.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:00.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:00.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:36:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:00.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:02.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:02.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:02.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:02.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:02.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:02.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:02.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:02.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:02.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:02.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:36:02.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:02.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:36:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 36 2026-03-23T19:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:36:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:36:06.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:06.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:06.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:36:06.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:36:06.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:06.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:36:06.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:06.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:36:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=36 2026-03-23T19:36:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:36:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:36:06.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:36:06.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:06.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:36:06.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=36 2026-03-23T19:36:06.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 36 = 36 2026-03-23T19:36:06.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:06.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:06.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:07.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:07.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:04 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294558,"replay_state":"idle"} 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:00 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 36 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.c60d01dd-c4e3-4c93-b639-8e8143a02a60 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:07.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:36:07.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:07.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:07.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:04' 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:36:04' 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:04' 2026-03-23T19:36:07.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:07.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:07.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:36:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294558,"replay_state":"idle"}2026-03-23 19:36:00' 2026-03-23T19:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:07.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:04' 2026-03-23T19:36:07.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:36:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:08.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:08.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:08.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294558,"replay_state":"idle"} 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:00 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:04' 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:36:08.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:08.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294558,"replay_state":"idle"} 2026-03-23T19:36:08.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:08.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:08.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:08.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:08.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:08.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:08.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:08.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:00' 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:36:00' 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:00' 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:08.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:08.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294558,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:36:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:36:04' 2026-03-23T19:36:08.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:08.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:08.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:00' 2026-03-23T19:36:08.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:08.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:36:09.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:09.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:10.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:10.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:10.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:10.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:10.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:10.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:10.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:10.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:10.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:10.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:10.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:10.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:10.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:10.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:10.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:10.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:10.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:10.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:10.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:10.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:36:10.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:10.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:12.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:12.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:12.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:12.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:12.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:12.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:12.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:12.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:12.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:12.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:12.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:12.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:12.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:12.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:13.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:10 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:13.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:09' 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:13.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:36:13.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:13.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:13.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:13.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:13.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:10' 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:36:10' 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:10' 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:13.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:13.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:13.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:36:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:36:09' 2026-03-23T19:36:13.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:13.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:13.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:10' 2026-03-23T19:36:13.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:13.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:13.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:13.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:13.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:13.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:14.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:09 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:10' 2026-03-23T19:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:14.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:14.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:14.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:14.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:14.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:14.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:14.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:09' 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:36:09' 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:09' 2026-03-23T19:36:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:36:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:36:10' 2026-03-23T19:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:14.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:09' 2026-03-23T19:36:14.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:14.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:14.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:14.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:14.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:36:14.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:36:15.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:16.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:36:16.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:16.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:16.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:16.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:16.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:16.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:36:19.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 39 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:36:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:36:19.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:19.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:19.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:36:19.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:36:19.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:36:19.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:36:19.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=39 2026-03-23T19:36:19.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:36:19.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:36:19.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:36:19.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:19.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:36:19.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=39 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 39 = 39 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:19.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:20.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:20 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294576,"remote_snapshot_timestamp":1774294576,"replay_state":"idle"} 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:19 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 39 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.bc2d79b6-5f9a-48d6-93e6-beaf7f245d6e (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:36:21.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:36:21.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:36:21.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:21.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:36:21.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:21.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:21.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:21.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:21.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:21.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:21.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:21.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:20' 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:36:20' 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:20' 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:21.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:21.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:36:20cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294576,"remote_snapshot_timestamp":1774294576,"replay_state":"idle"}2026-03-23 19:36:19' 2026-03-23T19:36:21.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:21.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:21.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:20' 2026-03-23T19:36:21.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:21.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:36:21.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:21.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:22.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:22.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:22.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:22.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294576,"remote_snapshot_timestamp":1774294576,"replay_state":"idle"} 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:19 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:20' 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:36:22.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:22.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294576,"remote_snapshot_timestamp":1774294576,"replay_state":"idle"} 2026-03-23T19:36:22.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:22.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:22.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:22.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:22.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:22.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:22.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:22.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:19' 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:36:19' 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:22.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:22.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:19' 2026-03-23T19:36:22.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:22.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:22.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294576,"remote_snapshot_timestamp":1774294576,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:36:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:36:20' 2026-03-23T19:36:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:22.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:19' 2026-03-23T19:36:22.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:36:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:22.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:36:22.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:36:22.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:22.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:23.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:23.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:23.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:23.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:23.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:23.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:23.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:36:23.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:23.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:25.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:25.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:25.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:25.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:25.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:25.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:25.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:25.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:25.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:25.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:25.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:26.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:25 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:24' 2026-03-23T19:36:26.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:26.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:26.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:26.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:26.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:26.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:26.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:26.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:26.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:27.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:27.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:27.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:25' 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:36:25' 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:25' 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:27.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:27.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:36:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:36:24' 2026-03-23T19:36:27.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:27.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:27.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:25' 2026-03-23T19:36:27.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:27.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:27.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:27.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:27.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:28.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:28.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:28.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:24 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:25' 2026-03-23T19:36:28.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:28.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:28.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:28.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:28.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:28.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:28.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:28.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:24' 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:36:24' 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:24' 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:28.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:28.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:36:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:36:25' 2026-03-23T19:36:28.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:28.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:28.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:24' 2026-03-23T19:36:28.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:28.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:28.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:28.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:36:28.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:36:29.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:29.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:30.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:30.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:30.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:30.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:30.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:30.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:30.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:30.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:30.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:30.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:30.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:30.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:30.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:30.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:30.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:36:30.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:30.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:32.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:32.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:32.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:32.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:32.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:32.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:32.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:32.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:32.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:32.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:32.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:32.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:32.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:32.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:36:35.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 42 2026-03-23T19:36:35.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:36:35.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:35.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:35.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:36:35.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:36:35.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:36:35.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:36:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=42 2026-03-23T19:36:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:36:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:36:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:36:35.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:35.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=42 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 42 = 42 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:35.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:36.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:34 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294588,"replay_state":"idle"} 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:30 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 42 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.9d182bb7-7d34-4cbb-98dc-2823adcb4102 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:36:36.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:36:36.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:36:36.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:36.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:36:36.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:36.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:36.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:36.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:36.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:36.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:36.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:34' 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:36:34' 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:34' 2026-03-23T19:36:36.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:36.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:36.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:36:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294588,"replay_state":"idle"}2026-03-23 19:36:30' 2026-03-23T19:36:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:36.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:34' 2026-03-23T19:36:36.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:36:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:36.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:36:36.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:36.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:36.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:36.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:37.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294588,"replay_state":"idle"} 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:30 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:34' 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:36:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:37.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294588,"replay_state":"idle"} 2026-03-23T19:36:37.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:37.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:37.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:37.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:37.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:30' 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:36:30' 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:30' 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:37.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294588,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:36:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:36:34' 2026-03-23T19:36:37.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:37.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:37.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:30' 2026-03-23T19:36:37.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:37.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:36:37.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:37.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:36:37.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:37.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:36:38.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:38.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:39.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:39.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:39.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:39.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:39.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:39.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:39.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:39.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:39.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:39.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:39.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:39.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:39.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:39.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:39.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:39.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:36:39.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:39.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:41.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:41.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:41.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:41.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:36:41.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:41.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:41.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:36:41.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:41.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:41.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:41.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:41.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:41.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:41.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:42.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:40 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:39' 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:42.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:42.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:42.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:42.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:42.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:42.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:42.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:42.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:42.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:42.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:42.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:40' 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:36:40' 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:40' 2026-03-23T19:36:42.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:42.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:42.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:42.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:36:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:36:39' 2026-03-23T19:36:42.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:42.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:40' 2026-03-23T19:36:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:42.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:42.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:42.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:42.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:43.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:39 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:40' 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:43.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:43.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:43.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:43.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:43.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:43.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:43.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:43.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:43.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:43.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:39' 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:36:39' 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:39' 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:43.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:43.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:36:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:36:40' 2026-03-23T19:36:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:43.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:39' 2026-03-23T19:36:43.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:43.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:36:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:36:45.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:45.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:46.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:46.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:46.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:46.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:46.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:46.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:46.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:46.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:46.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:46.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:36:46.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:36:46.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:46.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:46.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:46.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:36:49.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 45 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:49.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:36:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:36:49.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:49.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:49.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:36:49.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:36:49.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:36:49.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:36:49.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:49.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:36:49.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=45 2026-03-23T19:36:49.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:36:49.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:36:49.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:36:49.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:36:49.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=45 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 45 = 45 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:50.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:50 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294606,"remote_snapshot_timestamp":1774294606,"replay_state":"idle"} 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:49 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 45 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.84496e1c-cc7b-4b78-b244-c6ba8d8790fd (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:36:51.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:36:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:36:51.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:36:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:51.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:51.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:51.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:51.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:51.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:51.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:50' 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:36:50' 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:50' 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:51.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:51.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:36:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294606,"remote_snapshot_timestamp":1774294606,"replay_state":"idle"}2026-03-23 19:36:49' 2026-03-23T19:36:51.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:51.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:51.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:50' 2026-03-23T19:36:51.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:51.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:36:51.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:36:51.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:36:51.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:51.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:52.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:52.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:52.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:52.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294606,"remote_snapshot_timestamp":1774294606,"replay_state":"idle"} 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:49 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:50' 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:36:52.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:36:52.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:52.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294606,"remote_snapshot_timestamp":1774294606,"replay_state":"idle"} 2026-03-23T19:36:52.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:52.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:52.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:52.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:52.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:52.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:52.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:52.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:49' 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:36:49' 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:49' 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:52.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294606,"remote_snapshot_timestamp":1774294606,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:36:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:36:50' 2026-03-23T19:36:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:52.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:49' 2026-03-23T19:36:52.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:52.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:36:52.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:36:52.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:36:52.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:36:52.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:52.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:36:52.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:52.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:36:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:52.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:53.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:53.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:53.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:53.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:53.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:53.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:36:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:36:53.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:53.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:53.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:55.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:36:55.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:36:55.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:36:55.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:36:55.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:36:55.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:36:55.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:36:55.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:36:55.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:36:55.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:55.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:55.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:36:55.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:36:55.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:36:55.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:55.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:56.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:55 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:54' 2026-03-23T19:36:56.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:56.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:56.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:56.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:56.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:56.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:56.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:56.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:56.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:56.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:56.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:55' 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:36:55' 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:55' 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:36:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:36:54' 2026-03-23T19:36:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:57.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:57.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:55' 2026-03-23T19:36:57.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:57.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:57.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:57.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:57.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:58.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:54 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:36:55' 2026-03-23T19:36:58.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:36:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:36:58.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:36:58.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:36:58.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:36:58.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:58.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:36:58.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:58.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:36:58.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:36:58.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:36:58.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:36:58.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:54' 2026-03-23T19:36:58.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:36:54' 2026-03-23T19:36:58.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:36:54' 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:36:58.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:36:58.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:36:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:36:55' 2026-03-23T19:36:58.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:36:58.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:36:58.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:36:54' 2026-03-23T19:36:58.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:36:58.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:36:58.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:36:58.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:36:58.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:36:59.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:36:59.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:36:59.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:00.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:00.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:00.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:00.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:00.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:00.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:00.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:00.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:00.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:00.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:00.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:00.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:37:00.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:00.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:02.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:02.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:02.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:02.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:02.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:02.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:02.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:02.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:02.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:02.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:02.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:02.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:02.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:02.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:02.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:37:02.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:02.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:37:05.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 48 2026-03-23T19:37:05.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:37:05.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:37:05.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:05.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:05.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:05.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:05.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:37:05.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:37:05.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:05.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:05.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:37:05.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:37:05.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:37:05.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:05.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:37:05.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=48 2026-03-23T19:37:05.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:37:05.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:37:05.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:37:06.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:06.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:06.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:06.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:06.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:06.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:37:06.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:06.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=48 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 48 = 48 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:06.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:07.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:04 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:07.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294618,"replay_state":"idle"} 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:00 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 48 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.e521f199-6fce-454d-aa63-005ddc9970c9 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:37:07.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:07.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:37:07.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:07.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:07.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:07.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:07.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:07.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:07.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:07.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:04' 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:37:04' 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:04' 2026-03-23T19:37:07.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:07.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:07.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:07.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:37:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294618,"replay_state":"idle"}2026-03-23 19:37:00' 2026-03-23T19:37:07.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:07.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:07.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:04' 2026-03-23T19:37:07.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:07.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:37:07.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:07.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:37:07.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:07.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:08.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:08.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:08.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294618,"replay_state":"idle"} 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:00 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:08.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:08.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:08.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:04' 2026-03-23T19:37:08.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:37:08.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:37:08.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:08.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294618,"replay_state":"idle"} 2026-03-23T19:37:08.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:08.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:08.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:08.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:08.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:08.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:08.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:08.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:00' 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:37:00' 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:00' 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:08.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:08.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294618,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:37:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:37:04' 2026-03-23T19:37:08.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:08.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:08.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:00' 2026-03-23T19:37:08.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:08.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:37:08.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:08.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:37:08.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:08.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:37:08.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:08.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:08.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:09.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:09.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:09.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:09.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:09.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:09.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:09.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:09.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:09.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:09.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:09.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:09.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:09.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:09.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:09.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:37:09.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:09.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:11.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:11.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:11.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:12.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:12.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:13.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:13.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:13.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:10 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:09' 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:13.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:13.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:37:13.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:13.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:13.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:13.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:10' 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:37:10' 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:10' 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:13.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:13.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:37:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:37:09' 2026-03-23T19:37:13.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:13.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:10' 2026-03-23T19:37:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:13.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:13.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:13.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:13.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:13.154 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:14.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:09 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:10' 2026-03-23T19:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:37:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:14.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:14.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:14.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:14.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:14.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:09' 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:37:09' 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:09' 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:14.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:37:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:37:10' 2026-03-23T19:37:14.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:14.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:14.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:09' 2026-03-23T19:37:14.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:37:14.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:37:15.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:15.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:16.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:16.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:16.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:16.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:16.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:16.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:16.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:16.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:16.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:16.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:16.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:16.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:16.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:37:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:37:18.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:18.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:18.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:18.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:18.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:18.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:18.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:18.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:18.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:18.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:18.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:18.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:18.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:37:18.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:18.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:22.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:22.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:22.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:22.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:22.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:22.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:22.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:22.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:22.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:22.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:22.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:22.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:37:25.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 51 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:37:25.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:25.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:25.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:37:25.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:37:25.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:37:25.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:25.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:37:25.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=51 2026-03-23T19:37:25.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:37:25.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:37:25.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:37:25.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:25.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=51 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 51 = 51 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:25.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:26.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:26.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:26.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:25 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294634,"replay_state":"idle"} 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:19 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 51 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.e408aacc-6080-47e5-8507-26d11934b7ea (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:37:26.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:37:26.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:26.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:37:26.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:26.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:26.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:26.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:26.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:26.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:26.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:25' 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:37:25' 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:25' 2026-03-23T19:37:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:26.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:26.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:37:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294634,"replay_state":"idle"}2026-03-23 19:37:19' 2026-03-23T19:37:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:26.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:25' 2026-03-23T19:37:26.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:26.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:37:26.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:26.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:37:26.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:27.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:27.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:27.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294634,"replay_state":"idle"} 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:19 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:25' 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:37:27.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:27.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294634,"replay_state":"idle"} 2026-03-23T19:37:27.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:27.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:27.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:27.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:27.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:27.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:27.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:27.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:19' 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:37:19' 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:19' 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:27.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:27.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294634,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:37:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:37:25' 2026-03-23T19:37:27.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:27.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:27.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:19' 2026-03-23T19:37:27.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:27.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:37:27.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:27.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:27.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:37:28.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:28.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:29.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:29.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:29.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:29.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:29.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:29.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:29.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:29.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:29.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:29.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:29.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:29.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:29.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:37:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:31.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:31.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:31.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:31.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:31.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:31.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:31.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:31.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:31.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:32.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:32.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:32.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:32.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:32.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:32.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:32.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:32.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:32.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:30 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:30' 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:37:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:32.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:32.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:32.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:32.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:32.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:30' 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:37:30' 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:30' 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:32.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:32.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:37:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:37:30' 2026-03-23T19:37:32.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:32.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:32.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:30' 2026-03-23T19:37:32.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:32.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:32.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:32.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:32.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:32.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:33.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:30 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:30' 2026-03-23T19:37:33.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:33.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:33.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:33.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:37:33.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:33.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:33.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:33.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:33.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:33.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:33.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:33.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:30' 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:37:30' 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:30' 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:33.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:33.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:33.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted45124515mirror.0vm08.local2026-03-23 19:37:30cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:37:30' 2026-03-23T19:37:33.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:33.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:33.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:30' 2026-03-23T19:37:33.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:33.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:33.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:33.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:33.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:37:33.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:37:35.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:35.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:35.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:37:35.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:35.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:35.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:36.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:36.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:36.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:36.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:36.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:36.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:36.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:36.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:36.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:36.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:36.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:36.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:36.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:36.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:36.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:37:39.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 54 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:39.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:37:39.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:37:39.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:39.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:39.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:37:39.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:37:39.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:39.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:39.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:37:39.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:39.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:37:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=54 2026-03-23T19:37:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:37:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:37:39.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:37:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:39.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=54 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 54 = 54 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:39.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:39.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:39.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:39.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:39.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:40.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:39 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294654,"replay_state":"idle"} 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:35 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 54 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.5af828c8-fe6c-4c77-9e27-6c05dbafcab1 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:37:40.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:37:40.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:37:40.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:40.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:37:40.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:40.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:40.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:40.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:40.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:40.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:39' 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:37:39' 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:40.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:39' 2026-03-23T19:37:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:40.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:40.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:37:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294654,"replay_state":"idle"}2026-03-23 19:37:35' 2026-03-23T19:37:40.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:40.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:40.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:39' 2026-03-23T19:37:40.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:40.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:37:40.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:40.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:40.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:41.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294654,"replay_state":"idle"} 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:35 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:39' 2026-03-23T19:37:41.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:37:41.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:37:41.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:41.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294654,"replay_state":"idle"} 2026-03-23T19:37:41.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:41.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:41.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:41.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:41.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:41.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:41.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:41.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:35' 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:37:35' 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:35' 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:41.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:41.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294654,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:37:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:37:39' 2026-03-23T19:37:41.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:41.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:41.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:35' 2026-03-23T19:37:41.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:41.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:37:41.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:41.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:37:42.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:42.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:43.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:43.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:43.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:43.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:43.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:43.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:43.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:43.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:43.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:43.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:43.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:37:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:43.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:37:45.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:37:45.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:37:45.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:45.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:45.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:37:45.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:45.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:37:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:37:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:45.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:46.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:46.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:45 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:44' 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:46.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:46.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:37:46.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:46.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:46.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:46.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:46.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:46.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:46.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:46.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:45' 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:37:45' 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:45' 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:46.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:46.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:37:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:37:44' 2026-03-23T19:37:46.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:46.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:46.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:45' 2026-03-23T19:37:46.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:46.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:46.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:46.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:46.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:46.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:44 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:45' 2026-03-23T19:37:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:37:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:37:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:37:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:44' 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:37:44' 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:44' 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:47.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:47.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:37:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:37:45' 2026-03-23T19:37:47.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:47.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:47.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:44' 2026-03-23T19:37:47.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:47.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:37:47.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:37:47.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:37:49.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:49.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:50.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:50.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:50.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:50.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:50.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:50.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:50.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:50.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:50.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:50.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:50.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:50.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:37:50.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:37:50.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:37:50.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:50.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:50.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:50.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:37:54.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 57 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:37:54.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:54.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:37:54.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:37:54.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:54.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:54.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:37:54.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:37:54.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:37:54.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:54.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:37:54.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=57 2026-03-23T19:37:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:37:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:37:54.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:37:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:37:54.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=57 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 57 = 57 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:54.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:55.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:55.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:37:55.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:55.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:55 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294668,"replay_state":"idle"} 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:49 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 57 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.fac3fafa-c802-4480-89c6-db5337948fed (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:37:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:55.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:37:55.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:55.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:55.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:55.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:55.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:55.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:55.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:55.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:55' 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:37:55' 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:55' 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:55.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:55.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:37:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294668,"replay_state":"idle"}2026-03-23 19:37:49' 2026-03-23T19:37:55.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:55.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:55' 2026-03-23T19:37:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:37:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:37:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:55.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:56.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:56.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294668,"replay_state":"idle"} 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:49 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:55' 2026-03-23T19:37:56.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:37:56.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:37:56.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:37:56.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294668,"replay_state":"idle"} 2026-03-23T19:37:56.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:37:56.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:56.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:37:56.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:56.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:37:56.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:37:56.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:37:56.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:37:56.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:49' 2026-03-23T19:37:56.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:37:49' 2026-03-23T19:37:56.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:56.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:56.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:56.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:37:56.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:49' 2026-03-23T19:37:56.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:37:56.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:37:56.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:37:56.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294668,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:37:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:37:55' 2026-03-23T19:37:56.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:37:56.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:37:56.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:49' 2026-03-23T19:37:56.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:37:56.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:37:56.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:37:56.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:37:56.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:37:56.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:56.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:37:56.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:37:56.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:37:56.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:56.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:56.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:37:57.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:37:57.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:37:57.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:57.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:37:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:37:58.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:37:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:37:58.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:37:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:37:58.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:37:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:37:58.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:37:58.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:37:58.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:37:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:37:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:37:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:37:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:37:58.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:00.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:00.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:00.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:00.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:00.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:00.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:00.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:00.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:00.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:00.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:00.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:00.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:00.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:01.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:01.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:00 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:01.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:59' 2026-03-23T19:38:01.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:01.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:01.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:01.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:01.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:01.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:01.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:01.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:01.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:01.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:01.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:01.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:00' 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:38:00' 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:01.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:00' 2026-03-23T19:38:01.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:01.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:01.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:01.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:38:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:37:59' 2026-03-23T19:38:01.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:01.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:01.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:00' 2026-03-23T19:38:01.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:01.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:01.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:01.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:01.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:01.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:02.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:37:59 2026-03-23T19:38:02.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:00' 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:02.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:02.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:02.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:02.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:02.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:02.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:02.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:02.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:59' 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:37:59' 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:37:59' 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:02.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:02.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:37:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:38:00' 2026-03-23T19:38:02.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:02.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:02.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:37:59' 2026-03-23T19:38:02.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:02.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:02.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:02.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:38:02.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:38:04.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:38:04.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:04.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:05.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:05.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:05.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:05.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:05.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:05.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:05.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:05.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:05.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:05.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:05.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:05.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:07.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:07.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:07.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:07.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:07.238 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:07.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:07.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:07.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:07.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:07.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:07.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:07.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:07.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:07.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:38:11.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 60 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:38:11.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:11.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:11.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:38:11.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:38:11.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:11.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:11.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:38:11.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:11.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:38:11.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=60 2026-03-23T19:38:11.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:38:11.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:38:11.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:38:11.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:11.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=60 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 60 = 60 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:11.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:12.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:12.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:12.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:09 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294683,"replay_state":"idle"} 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:05 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 60 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.8bcd37dd-9fe9-45f1-a83d-3e512864640b (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:38:12.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:38:12.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:38:12.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:12.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:38:12.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:12.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:12.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:12.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:12.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:12.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:12.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:12.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:09' 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:38:09' 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:09' 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:12.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:38:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294683,"replay_state":"idle"}2026-03-23 19:38:05' 2026-03-23T19:38:12.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:12.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:12.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:09' 2026-03-23T19:38:12.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:12.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:38:12.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:12.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:38:12.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:12.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:13.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294683,"replay_state":"idle"} 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:05 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:09' 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:38:13.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:13.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294683,"replay_state":"idle"} 2026-03-23T19:38:13.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:13.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:13.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:13.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:13.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:13.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:13.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:13.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:05' 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:38:05' 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:05' 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:13.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294683,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:38:05cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:38:09' 2026-03-23T19:38:13.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:13.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:13.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:05' 2026-03-23T19:38:13.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:13.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:38:14.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:14.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:15.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:15.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:15.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:15.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:15.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:15.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:15.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:15.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:15.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:15.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:15.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:15.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:15.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:16.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:16.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:15 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:14' 2026-03-23T19:38:16.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:16.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:16.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:16.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:16.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:16.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:16.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:16.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:16.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:16.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:16.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:15' 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:38:15' 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:15' 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:16.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:16.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:38:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:38:14' 2026-03-23T19:38:16.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:16.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:16.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:15' 2026-03-23T19:38:16.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:16.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:17.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:14 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:17.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:15' 2026-03-23T19:38:17.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:17.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:17.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:17.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:17.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:17.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:17.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:17.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:17.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:17.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:17.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:17.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:14' 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:38:14' 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:17.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:14' 2026-03-23T19:38:17.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:17.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:17.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:17.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:38:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:38:15' 2026-03-23T19:38:17.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:17.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:17.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:14' 2026-03-23T19:38:17.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:17.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:17.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:17.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:38:17.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:38:18.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:18.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:19.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:19.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:19.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:19.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:19.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:19.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:19.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:19.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:19.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:19.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:38:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:38:21.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:21.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:21.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:21.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:21.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:21.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:21.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:21.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:21.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:21.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:21.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:21.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:21.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:21.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:21.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:38:24.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 63 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:38:24.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:24.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:24.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:38:24.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:38:24.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:24.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:38:24.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:38:24.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=63 2026-03-23T19:38:24.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:38:24.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:38:24.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:38:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:38:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:24.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=63 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 63 = 63 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:24.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:25.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:25 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294697,"replay_state":"idle"} 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:19 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 62 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.43b70f35-1f2d-4ea5-bd5d-ef8a5ead0157 (peer_uuids:[]) 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 63 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.826733b0-84db-4c06-929c-aaf59310d246 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:38:25.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:38:25.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:38:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:25.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:25.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:25' 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:38:25' 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:25' 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:25.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:25.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:38:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294697,"replay_state":"idle"}2026-03-23 19:38:19' 2026-03-23T19:38:25.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:25.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:25.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:25' 2026-03-23T19:38:25.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:25.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:38:25.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:25.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:25.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:26.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:26.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:26.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:26.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294697,"replay_state":"idle"} 2026-03-23T19:38:26.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:19 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:25' 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:38:26.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:26.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294697,"replay_state":"idle"} 2026-03-23T19:38:26.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:26.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:26.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:26.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:26.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:26.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:26.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:26.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:19' 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:38:19' 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:19' 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:26.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294697,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:38:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:38:25' 2026-03-23T19:38:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:26.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:26.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:19' 2026-03-23T19:38:26.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:38:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:26.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:38:26.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:26.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:38:28.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:38:28.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:38:28.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:28.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:28.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:28.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:28.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:29.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:29.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:29.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:29.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:29.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:29.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:29.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:29.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:29.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:29.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:29.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:29.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:29.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:38:29.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:29.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:30.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:30 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:29' 2026-03-23T19:38:30.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:30.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:30.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:30.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:30.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:30.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:30.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:30.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:30.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:30.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:30.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:30.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:30' 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:38:30' 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:30' 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:30.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:38:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:38:29' 2026-03-23T19:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:30.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:30' 2026-03-23T19:38:30.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:30.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:30.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:30.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:30.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:31.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:38:31.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:31.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:29 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:30' 2026-03-23T19:38:31.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:31.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:31.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:31.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:31.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:31.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:31.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:31.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:31.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:31.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:31.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:31.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:29' 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:38:29' 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:29' 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:31.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:38:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:38:30' 2026-03-23T19:38:31.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:31.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:29' 2026-03-23T19:38:31.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:31.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:31.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:31.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:38:31.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:38:33.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:38:33.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:33.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:34.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:34.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:34.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:34.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:34.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:34.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:34.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:34.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:34.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:34.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:34.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:34.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:34.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:34.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:38:34.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:34.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:38:36.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:36.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:36.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:36.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:36.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:36.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:36.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:36.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:36.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:36.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:36.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:36.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:36.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:38:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 66 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:38:40.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:40.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:40.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:38:40.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:38:40.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:38:40.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:38:40.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=66 2026-03-23T19:38:40.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:38:40.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:38:40.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:38:40.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:40.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:40.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:38:40.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=66 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 66 = 66 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:40.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:41.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:41.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:41.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:41.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:39 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294712,"replay_state":"idle"} 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:35 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 66 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.8a5919f8-c05e-44c5-a085-ee9ab4b232f5 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:38:41.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:41.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:38:41.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:41.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:41.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:41.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:41.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:41.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:41.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:41.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:41.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:39' 2026-03-23T19:38:41.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:38:39' 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:39' 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:41.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:41.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:38:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294712,"replay_state":"idle"}2026-03-23 19:38:35' 2026-03-23T19:38:41.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:41.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:41.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:39' 2026-03-23T19:38:41.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:41.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:38:41.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:41.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:38:41.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:41.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:41.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:42.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:42.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294712,"replay_state":"idle"} 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:35 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:39' 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:38:42.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:38:42.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:42.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294712,"replay_state":"idle"} 2026-03-23T19:38:42.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:42.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:42.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:42.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:42.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:42.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:42.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:42.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:35' 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:38:35' 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:35' 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:42.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:42.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294712,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:38:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:38:39' 2026-03-23T19:38:42.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:42.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:35' 2026-03-23T19:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:42.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:38:42.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:42.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:38:43.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:43.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:44.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:44.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:44.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:44.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:44.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:44.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:44.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:44.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:38:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:46.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:46.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:38:46.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:46.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:46.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:38:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:38:46.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:46.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:46.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:38:46.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:38:46.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:47.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:47.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:45 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:44' 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:47.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:47.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:47.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:47.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:47.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:45' 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:38:45' 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:45' 2026-03-23T19:38:47.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:47.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:47.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:38:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:38:44' 2026-03-23T19:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:45' 2026-03-23T19:38:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:48.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:38:48.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:48.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:48.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:44 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:45' 2026-03-23T19:38:48.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:38:48.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:38:48.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:48.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:38:48.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:48.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:48.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:48.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:48.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:48.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:48.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:48.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:44' 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:38:44' 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:44' 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:48.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:48.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:38:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:38:45' 2026-03-23T19:38:48.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:48.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:48.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:44' 2026-03-23T19:38:48.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:48.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:38:48.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:38:48.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:38:48.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:50.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:51.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:51.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:51.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:51.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:51.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:51.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:51.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:51.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:51.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:51.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:51.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:38:55.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 69 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:38:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:55.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:38:55.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:38:55.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:55.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:55.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:38:55.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:38:55.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:38:55.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:38:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:55.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:38:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=69 2026-03-23T19:38:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:38:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:38:55.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:38:55.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:38:55.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=69 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 69 = 69 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:55.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:55.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:56.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:55 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294729,"replay_state":"idle"} 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:49 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 69 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.5f3434c4-0860-46b4-a77e-8ba01f04eb88 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:38:56.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:38:56.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:38:56.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:56.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:38:56.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:56.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:56.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:56.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:56.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:56.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:56.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:56.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:55' 2026-03-23T19:38:56.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:38:55' 2026-03-23T19:38:56.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:55' 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:56.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:56.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:38:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294729,"replay_state":"idle"}2026-03-23 19:38:49' 2026-03-23T19:38:56.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:56.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:55' 2026-03-23T19:38:56.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:38:56.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:38:56.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:38:57.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294729,"replay_state":"idle"} 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:49 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:38:55' 2026-03-23T19:38:57.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:38:57.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:38:57.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:38:57.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294729,"replay_state":"idle"} 2026-03-23T19:38:57.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:38:57.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:57.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:38:57.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:57.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:38:57.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:38:57.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:38:57.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:49' 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:38:49' 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:38:49' 2026-03-23T19:38:57.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:38:57.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:38:57.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:38:57.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294729,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:38:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:38:55' 2026-03-23T19:38:57.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:38:57.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:38:57.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:38:49' 2026-03-23T19:38:57.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:38:57.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:38:57.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:38:57.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:57.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:38:58.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:58.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:38:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:38:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:38:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:38:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:38:59.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:59.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:38:59.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:38:59.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:38:59.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:38:59.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:38:59.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:38:59.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:38:59.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:38:59.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:38:59.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:38:59.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:38:59.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:38:59.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:38:59.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:38:59.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:38:59.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:39:00.203 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:39:00.200+0000 7f36e2336640 -1 reset not still connected to 0x55d738a8aa90 2026-03-23T19:39:00.203 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:39:00.201+0000 7f36e2336640 -1 reset not still connected to 0x55d738a8b860 2026-03-23T19:39:00.203 INFO:tasks.ceph.cluster2.osd.0.vm08.stderr:2026-03-23T19:39:00.201+0000 7f36e2336640 -1 reset not still connected to 0x55d738eac000 2026-03-23T19:39:01.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:01.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:01.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:01.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:01.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:01.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:03.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:00 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:00' 2026-03-23T19:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:03.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:03.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:03.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:03.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:00' 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:39:00' 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:00' 2026-03-23T19:39:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:03.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:39:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:39:00' 2026-03-23T19:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:03.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:00' 2026-03-23T19:39:03.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:03.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:03.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:03.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:03.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:03.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:03.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:03.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:03.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:04.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:04.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:00 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:00' 2026-03-23T19:39:04.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:04.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:04.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:04.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:39:04.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:04.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:04.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:04.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:04.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:04.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:04.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:04.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:04.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:00' 2026-03-23T19:39:04.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:39:00' 2026-03-23T19:39:04.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:04.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:04.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:04.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:04.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:00' 2026-03-23T19:39:04.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:04.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:04.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:04.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted45124515mirror.0vm08.local2026-03-23 19:39:00cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:39:00' 2026-03-23T19:39:04.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:04.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:04.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:00' 2026-03-23T19:39:04.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:04.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:04.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:04.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:04.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:39:04.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:39:06.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:06.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:07.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:07.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:07.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:07.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:07.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:07.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:07.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:07.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:07.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:07.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:07.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:07.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:07.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:07.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:07.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:39:11.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 72 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:11.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:39:11.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:39:11.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:11.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:11.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:39:11.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:39:11.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:11.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:39:11.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:11.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:39:11.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=72 2026-03-23T19:39:11.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:39:11.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:39:11.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:39:11.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:11.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=72 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 72 = 72 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:11.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:12.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:09 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:12.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294745,"replay_state":"idle"} 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:05 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 72 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.99c7c074-8cf1-4dd1-8bef-dfab3b7bcd98 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:39:12.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:12.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:39:12.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:12.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:12.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:12.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:12.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:12.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:12.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:12.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:12.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:09' 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:39:09' 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:09' 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:12.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:12.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:39:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294745,"replay_state":"idle"}2026-03-23 19:39:05' 2026-03-23T19:39:12.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:12.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:09' 2026-03-23T19:39:12.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:39:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:39:12.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:12.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:39:13.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:13.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294745,"replay_state":"idle"} 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:05 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:09' 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:39:13.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:39:13.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:13.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294745,"replay_state":"idle"} 2026-03-23T19:39:13.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:13.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:13.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:13.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:13.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:13.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:13.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:13.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:13.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:05' 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:39:05' 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:05' 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:13.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294745,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:39:05cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:39:09' 2026-03-23T19:39:13.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:13.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:13.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:05' 2026-03-23T19:39:13.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:13.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:39:13.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:13.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:39:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:39:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:39:13.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:13.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:13.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:13.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:39:14.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:14.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:39:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:14.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:15.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:15.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:15.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:15.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:15.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:15.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:15.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:15.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:15.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:15.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:15.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:15.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:15.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:15.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:15.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:15.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:16.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:16.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:16.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:16.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:16.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:16.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:15 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:14' 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:16.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:16.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:16.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:16.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:16.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:15' 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:39:15' 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:15' 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:16.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:16.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:39:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:39:14' 2026-03-23T19:39:16.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:16.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:16.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:15' 2026-03-23T19:39:16.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:16.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:16.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:16.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:16.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:17.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:17.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:14 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:15' 2026-03-23T19:39:17.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:17.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:17.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:17.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:17.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:17.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:17.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:17.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:14' 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:39:14' 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:14' 2026-03-23T19:39:17.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:17.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:17.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:17.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:39:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:39:15' 2026-03-23T19:39:17.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:17.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:17.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:14' 2026-03-23T19:39:17.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:17.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:17.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:17.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:17.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:39:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:39:19.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:19.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:20.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:20.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:20.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:20.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:20.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:20.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:20.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:20.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:20.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:20.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:20.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:20.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:20.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:20.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:20.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:20.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:39:24.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 75 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:39:24.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:24.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:24.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:39:24.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:39:24.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:24.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:24.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:24.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:39:24.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:39:24.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=75 2026-03-23T19:39:24.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:39:24.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:39:24.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:39:25.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:25.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=75 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 75 = 75 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:25.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:25.280 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:39:25.278+0000 7effd6f28640 -1 reset not still connected to 0x562079b99110 2026-03-23T19:39:25.280 INFO:tasks.ceph.cluster2.osd.1.vm08.stderr:2026-03-23T19:39:25.278+0000 7effd6f28640 -1 reset not still connected to 0x56207ad62410 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:26.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:25 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294758,"replay_state":"idle"} 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:19 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 75 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.23163d71-f0b3-491f-aa6f-74f92a79a4bd (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:39:26.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:39:26.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:39:26.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:26.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:39:26.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:26.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:26.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:26.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:26.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:26.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:26.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:26.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:25' 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:39:25' 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:25' 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:26.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:39:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294758,"replay_state":"idle"}2026-03-23 19:39:19' 2026-03-23T19:39:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:26.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:26.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:25' 2026-03-23T19:39:26.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:39:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:27.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:27.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294758,"replay_state":"idle"} 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:19 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:25' 2026-03-23T19:39:27.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:27.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294758,"replay_state":"idle"} 2026-03-23T19:39:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:27.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:27.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:27.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:27.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:27.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:27.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:27.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:19' 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:39:19' 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:19' 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:27.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:27.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294758,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:39:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:39:25' 2026-03-23T19:39:27.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:27.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:27.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:19' 2026-03-23T19:39:27.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:39:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:27.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:39:27.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:27.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:39:27.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:39:27.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:39:27.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:27.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:28.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:28.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:28.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:28.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:28.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:28.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:28.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:28.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:28.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:28.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:28.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:28.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:28.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:28.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:28.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:28.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:39:28.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:28.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:30.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:30.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:30.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:30.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:30.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:30.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:30.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:30.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:30.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:30.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:30.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:31.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:31.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:31.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:31.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:31.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:32.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:30 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:29' 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:32.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:32.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:32.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:32.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:32.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:32.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:32.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:32.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:32.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:32.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:32.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:30' 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:39:30' 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:30' 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:32.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:32.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:39:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:39:29' 2026-03-23T19:39:32.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:32.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:32.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:30' 2026-03-23T19:39:32.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:32.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:32.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:32.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:32.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:32.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:29 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:30' 2026-03-23T19:39:33.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:33.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:33.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:33.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:33.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:33.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:33.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:33.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:33.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:33.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:29' 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:39:29' 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:29' 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:33.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:33.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:39:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:39:30' 2026-03-23T19:39:33.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:33.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:33.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:29' 2026-03-23T19:39:33.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:33.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:33.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:33.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:33.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:33.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:33.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:39:33.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:39:34.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:39:34.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:34.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:34.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:35.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:35.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:35.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:35.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:35.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:35.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:35.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:35.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:35.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:35.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:35.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:35.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:39:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:35.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:37.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:37.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:37.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:37.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:37.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:37.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:37.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:37.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:37.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:39:37.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:37.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:39:41.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 78 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:39:41.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:41.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:41.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:39:41.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:39:41.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:39:41.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:41.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:39:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=78 2026-03-23T19:39:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:39:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:39:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:39:41.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:39:41.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=78 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 78 = 78 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:41.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:42.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:42.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:42.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:39 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294773,"replay_state":"idle"} 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:35 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 78 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.bbf6e2c2-e14d-492b-9a80-a519926e5a20 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:39:42.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:39:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:39:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:42.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:39:42.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:42.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:42.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:42.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:42.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:42.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:39' 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:39:39' 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:39' 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:42.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:42.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:39:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294773,"replay_state":"idle"}2026-03-23 19:39:35' 2026-03-23T19:39:42.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:42.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:39' 2026-03-23T19:39:42.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:42.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:39:42.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:39:42.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:39:42.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:42.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:43.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:39:43.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:43.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294773,"replay_state":"idle"} 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:35 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:39' 2026-03-23T19:39:43.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:39:43.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:39:43.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:43.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294773,"replay_state":"idle"} 2026-03-23T19:39:43.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:43.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:43.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:43.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:43.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:43.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:43.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:43.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:43.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:35' 2026-03-23T19:39:43.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:39:35' 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:35' 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:43.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:43.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294773,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:39:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:39:39' 2026-03-23T19:39:43.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:43.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:35' 2026-03-23T19:39:43.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:39:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:39:43.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:39:43.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:43.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:39:44.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:45.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:45.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:45.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:45.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:45.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:45.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:45.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:45.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:45.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:45.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:45.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:45.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:45.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:45.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:39:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:45.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:39:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:39:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:39:47.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:47.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:39:47.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:47.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:47.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:39:47.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:47.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:48.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:48.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:48.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:48.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:48.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:45 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:44' 2026-03-23T19:39:48.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:48.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:48.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:48.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:39:48.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:48.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:48.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:48.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:48.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:48.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:48.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:48.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:48.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:45' 2026-03-23T19:39:48.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:39:45' 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:45' 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:48.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:48.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:39:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:39:44' 2026-03-23T19:39:48.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:48.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:48.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:45' 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:48.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:49.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:49 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:39:49.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:39:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:39:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:39:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:45' 2026-03-23T19:39:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:39:49.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:39:49.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:39:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:39:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:39:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:49.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:39:49.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:49.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:39:49.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:39:49.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:39:49.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:39:49.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:49' 2026-03-23T19:39:49.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:39:49' 2026-03-23T19:39:49.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:49.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:49' 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:39:49.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:39:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:39:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:39:45' 2026-03-23T19:39:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:39:49.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:39:49.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:49' 2026-03-23T19:39:49.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:39:49.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:39:49.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:39:49.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:39:49.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:39:51.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:51.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:39:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:39:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:52.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:52.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:52.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:52.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:52.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:52.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:52.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:52.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:52.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:52.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:52.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:52.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:52.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:52.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:39:52.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:52.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:39:54.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:39:54.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:54.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:54.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:54.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:54.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:54.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:54.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:54.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:54.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:54.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:54.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:54.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:54.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:54.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:54.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:39:54.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:39:54.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:39:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:54.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:39:58.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:39:58.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:39:58.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:39:58.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:39:58.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:39:58.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:39:58.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:39:58.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:39:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:39:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:39:58.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:39:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:39:58.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:39:58.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:39:58.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:39:58.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:39:58.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:39:58.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:39:58.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:39:58.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:39:58.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:39:58.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:40:01.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 81 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:40:01.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:40:01.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:01.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:01.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:40:01.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:40:01.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:01.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:01.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:40:01.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:01.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:40:01.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=81 2026-03-23T19:40:01.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:40:01.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:40:01.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:40:01.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:01.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=81 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 81 = 81 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:01.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:02.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:02.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:00 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294790,"replay_state":"idle"} 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:54 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 81 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.a2046365-b5d7-4a00-ad60-570a9b8a4748 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:40:02.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:40:02.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:02.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:40:02.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:02.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:02.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:02.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:02.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:02.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:02.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:02.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:00' 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:40:00' 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:00' 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:40:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294790,"replay_state":"idle"}2026-03-23 19:39:54' 2026-03-23T19:40:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:02.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:02.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:00' 2026-03-23T19:40:02.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:02.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:40:02.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:02.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:40:02.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:02.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294790,"replay_state":"idle"} 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:39:54 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:00' 2026-03-23T19:40:03.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:40:03.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:40:03.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294790,"replay_state":"idle"} 2026-03-23T19:40:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:03.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:03.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:03.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:54' 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:39:54' 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:39:54' 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:03.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294790,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:39:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:40:00' 2026-03-23T19:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:03.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:39:54' 2026-03-23T19:40:03.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:03.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:40:03.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:03.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:03.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:40:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:04.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:05.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:05.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:05.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:05.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:05.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:05.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:05.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:05.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:05.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:05.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:05.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:05.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:05.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:05.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:05.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:05.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:05.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:40:05.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:05.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:40:07.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:07.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:07.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:07.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:07.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:07.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:07.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:07.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:07.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:07.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:07.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:08.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:40:08.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:08.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:08.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:05 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:06' 2026-03-23T19:40:09.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:09.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:09.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:09.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:09.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:09.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:09.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:09.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:05' 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:40:05' 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:05' 2026-03-23T19:40:09.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:09.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:09.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:40:05cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:40:06' 2026-03-23T19:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:05' 2026-03-23T19:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:09.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:09.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:10.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:09 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:10.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:10.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:10.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:05' 2026-03-23T19:40:10.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:10.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:10.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:10.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:10.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:10.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:10.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:10.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:10.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:10.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:10.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:10.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:09' 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:40:09' 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:09' 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:10.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:40:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:40:05' 2026-03-23T19:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:10.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:10.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:09' 2026-03-23T19:40:10.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:10.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:10.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:10.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:10.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:40:10.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:40:11.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:11.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:12.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:12.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:12.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:12.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:12.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:12.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:12.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:12.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:12.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:12.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:40:15.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 84 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:40:15.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:15.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:15.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:40:15.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:40:15.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:40:15.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:15.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:40:15.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=84 2026-03-23T19:40:15.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:40:15.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:40:15.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:40:15.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:15.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:40:15.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=84 2026-03-23T19:40:15.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 84 = 84 2026-03-23T19:40:15.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:15.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:16.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:14 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294810,"replay_state":"idle"} 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:10 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 83 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.dede97cf-48ad-47eb-bee4-9bef6941d1d6 (peer_uuids:[]) 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 84 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.42945ec3-cd07-4406-bc74-84432369029b (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:40:16.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:40:16.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:40:16.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:16.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:40:16.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:16.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:16.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:16.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:16.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:14' 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:40:14' 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:16.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:14' 2026-03-23T19:40:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:40:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294810,"replay_state":"idle"}2026-03-23 19:40:10' 2026-03-23T19:40:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:16.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:16.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:14' 2026-03-23T19:40:16.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:40:16.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:40:16.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:16.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294810,"replay_state":"idle"} 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:10 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:14' 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:40:17.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:40:17.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:17.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294810,"replay_state":"idle"} 2026-03-23T19:40:17.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:17.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:17.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:17.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:17.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:17.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:17.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:17.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:10' 2026-03-23T19:40:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:40:10' 2026-03-23T19:40:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:17.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:17.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:10' 2026-03-23T19:40:17.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:17.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:17.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:17.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294810,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:40:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:40:14' 2026-03-23T19:40:17.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:17.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:17.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:10' 2026-03-23T19:40:17.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:17.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:40:17.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:17.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:17.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:40:19.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:40:19.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:19.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:20.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:20.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:20.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:20.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:20.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:20.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:20.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:20.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:20.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:21.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:20 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:19' 2026-03-23T19:40:21.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:21.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:21.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:21.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:21.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:21.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:21.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:21.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:21.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:21.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:21.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:21.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:20' 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:40:20' 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:20' 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:21.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:40:20cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:40:19' 2026-03-23T19:40:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:21.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:21.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:20' 2026-03-23T19:40:21.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:21.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:21.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:21.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:22.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:19 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:20' 2026-03-23T19:40:22.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:22.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:22.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:22.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:22.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:22.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:22.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:22.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:22.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:22.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:22.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:19' 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:40:19' 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:19' 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:22.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:22.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:40:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:40:20' 2026-03-23T19:40:22.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:22.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:22.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:19' 2026-03-23T19:40:22.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:22.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:22.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:22.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:40:22.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:40:24.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:24.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:25.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:40:25.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:25.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:25.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:25.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:25.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:25.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:25.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:25.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:25.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:25.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:25.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:25.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:25.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:25.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:25.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:25.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:25.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:25.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:25.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:25.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:25.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:25.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:40:29.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 87 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:40:29.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:29.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:29.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:40:29.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:40:29.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:40:29.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:29.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:40:29.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=87 2026-03-23T19:40:29.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:40:29.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:40:29.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:29.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:40:29.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:40:29.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=87 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 87 = 87 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:29.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:30.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:30.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:30 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294823,"replay_state":"idle"} 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:24 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 87 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.0a3c8897-d3c8-42ad-90a1-70da98f86ca9 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:40:30.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:40:30.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:40:30.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:30.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:40:30.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:30.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:30.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:30.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:30.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:30.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:30.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:30.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:30' 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:40:30' 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:30' 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:30.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:40:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294823,"replay_state":"idle"}2026-03-23 19:40:24' 2026-03-23T19:40:30.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:30.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:30.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:30' 2026-03-23T19:40:30.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:40:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:30.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:30.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:31.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:31.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:31.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294823,"replay_state":"idle"} 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:24 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:30' 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:40:31.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294823,"replay_state":"idle"} 2026-03-23T19:40:31.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:32.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:32.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:32.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:32.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:32.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:32.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:32.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:24' 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:40:24' 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:24' 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:32.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:32.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294823,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:40:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:40:30' 2026-03-23T19:40:32.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:32.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:32.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:24' 2026-03-23T19:40:32.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:32.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:40:32.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:32.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:40:32.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:32.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:40:32.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:32.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:40:32.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:32.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:32.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:33.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:33.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:33.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:33.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:33.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:33.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:33.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:33.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:33.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:33.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:33.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:33.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:33.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:33.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:33.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:40:33.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:33.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:35.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:35.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:35.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:35.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:35.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:35.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:35.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:35.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:35.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:35.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:36.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:35 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:34' 2026-03-23T19:40:36.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:36.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:36.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:36.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:36.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:36.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:35' 2026-03-23T19:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:40:35' 2026-03-23T19:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:35' 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:36.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:40:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:40:34' 2026-03-23T19:40:36.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:36.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:36.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:35' 2026-03-23T19:40:36.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:36.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:36.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:36.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:36.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:36.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:37.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:37.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:34 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:35' 2026-03-23T19:40:37.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:37.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:37.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:37.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:37.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:37.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:37.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:37.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:37.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:37.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:37.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:37.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:34' 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:40:34' 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:34' 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:37.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:40:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:40:35' 2026-03-23T19:40:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:37.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:37.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:34' 2026-03-23T19:40:37.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:37.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:37.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:37.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:40:37.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:40:39.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:39.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:40.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:40.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:40.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:40.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:40.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:40.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:40.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:40.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:40.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:40.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:40.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:40.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:40.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:40.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:40.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:40.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Starting"' 2026-03-23T19:40:40.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Starting"' 2026-03-23T19:40:40.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:40.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:40:40.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:40.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:42.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:42.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:42.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:42.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:42.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:42.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:42.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:42.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:42.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:42.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:42.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:42.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:42.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:42.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:42.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:42.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:42.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:42.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:42.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:42.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:40:42.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:42.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:42.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:40:46.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 90 2026-03-23T19:40:46.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:40:46.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:40:46.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:46.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:46.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:40:46.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:40:46.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:40:46.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:46.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:40:46.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=90 2026-03-23T19:40:46.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:40:46.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:40:46.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:40:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:40:46.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=90 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 90 = 90 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:46.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:47.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:47.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:44 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294838,"replay_state":"idle"} 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:40 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 90 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f2856ede-45f1-4585-b38e-c868a15c78b0 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:40:47.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:40:47.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:40:47.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:47.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:40:47.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:47.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:47.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:47.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:47.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:47.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:47.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:47.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:47.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:44' 2026-03-23T19:40:47.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:40:44' 2026-03-23T19:40:47.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:47.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:44' 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:47.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:40:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294838,"replay_state":"idle"}2026-03-23 19:40:40' 2026-03-23T19:40:47.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:47.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:47.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:44' 2026-03-23T19:40:47.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:40:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:40:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:47.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:48.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294838,"replay_state":"idle"} 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:40 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:44' 2026-03-23T19:40:48.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:40:48.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:40:48.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:48.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294838,"replay_state":"idle"} 2026-03-23T19:40:48.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:48.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:48.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:48.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:48.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:48.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:48.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:48.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:48.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:40' 2026-03-23T19:40:48.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:40:40' 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:40' 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:48.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294838,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:40:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:40:44' 2026-03-23T19:40:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:48.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:48.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:40' 2026-03-23T19:40:48.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:48.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:40:48.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:40:48.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:40:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:40:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:40:49.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:40:49.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:49.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:50.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:50.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:50.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:50.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:50.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:50.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:50.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:50.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:50.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:50.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:50.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:50.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:40:50.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:40:50.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:50.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:40:50.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:40:50.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:40:50.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:50.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:52.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:52.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:52.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:40:52.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:40:52.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:52.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:52.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:40:52.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:52.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:52.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:52.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:52.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:53.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:53.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:50 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:49' 2026-03-23T19:40:53.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:53.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:53.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:53.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:40:53.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:53.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:53.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:53.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:53.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:53.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:53.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:53.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:50' 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:40:50' 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:50' 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:53.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:53.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:40:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:40:49' 2026-03-23T19:40:53.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:53.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:53.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:50' 2026-03-23T19:40:53.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:53.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:53.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:53.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:53.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:53.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:40:54.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:54.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:54.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:40:54.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:40:54.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:40:54.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:54.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:49 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:50' 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:40:54.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:40:54.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:40:54.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:40:54.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:40:54.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:54.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:40:54.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:54.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:40:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:40:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:40:54.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:49' 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:40:49' 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:49' 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:40:54.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:40:54.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:40:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:40:50' 2026-03-23T19:40:54.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:40:54.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:40:54.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:49' 2026-03-23T19:40:54.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:40:54.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:40:54.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:40:54.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:40:54.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:40:54.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:40:56.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:57.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:57.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:57.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:57.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:57.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:57.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:57.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:57.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:57.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:57.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:57.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:57.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:57.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:40:57.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:57.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:40:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:40:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:40:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:40:59.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:40:59.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:59.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:59.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:40:59.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:40:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:40:59.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:40:59.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:40:59.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:40:59.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:40:59.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:40:59.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:03.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:03.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:03.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:03.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:03.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:03.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:03.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:03.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:03.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:03.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:03.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:03.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:41:06.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 93 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:06.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:06.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:41:06.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:41:06.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:06.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:06.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:41:06.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:41:06.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:41:06.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:06.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:41:06.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=93 2026-03-23T19:41:06.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:41:06.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:41:06.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:06.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:41:06.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:06.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=93 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 93 = 93 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:06.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:07.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:07.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:05 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294855,"replay_state":"idle"} 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:59 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 93 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f51138e4-3015-4b5d-b135-ed6d3dbb8892 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:41:07.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:07.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:41:07.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:07.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:07.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:07.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:07.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:07.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:07.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:07.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:05' 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:41:05' 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:05' 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:07.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:07.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:07.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:41:05cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294855,"replay_state":"idle"}2026-03-23 19:40:59' 2026-03-23T19:41:07.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:07.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:07.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:05' 2026-03-23T19:41:07.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:07.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:41:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:41:07.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:08.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:08.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294855,"replay_state":"idle"} 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:40:59 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:05' 2026-03-23T19:41:08.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:41:08.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:41:08.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:08.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294855,"replay_state":"idle"} 2026-03-23T19:41:08.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:08.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:08.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:08.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:08.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:08.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:08.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:59' 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:40:59' 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:40:59' 2026-03-23T19:41:08.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:08.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:08.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:08.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294855,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:40:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:41:05' 2026-03-23T19:41:08.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:08.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:08.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:40:59' 2026-03-23T19:41:08.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:08.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:41:08.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:08.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:41:08.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:08.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:08.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:41:08.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:08.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:09.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:09.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:09.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:09.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:10.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:10.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:10.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:10.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:10.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:10.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:10.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:10.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:10.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:10.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:10.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:41:10.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:10.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:12.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:12.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:12.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:12.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:12.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:12.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:12.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:41:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:41:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:12.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:13.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:13.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:13.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:13.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:13.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:13.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:13.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:10 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:10' 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:13.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:13.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:41:13.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:13.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:13.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:13.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:13.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:13.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:13.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:13.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:10' 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:41:10' 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:10' 2026-03-23T19:41:13.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:13.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:13.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:13.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:41:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:41:10' 2026-03-23T19:41:13.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:13.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:13.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:10' 2026-03-23T19:41:13.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:13.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:13.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:13.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:13.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:13.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:10 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:14.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:14.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:14.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:14.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:10' 2026-03-23T19:41:14.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:14.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:14.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:14.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:41:14.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:14.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:14.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:14.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:14.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:14.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:14.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:14.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:10' 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:41:10' 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:10' 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:14.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted45124515mirror.0vm08.local2026-03-23 19:41:10cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:41:10' 2026-03-23T19:41:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:14.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:14.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:10' 2026-03-23T19:41:14.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:41:14.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:41:16.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:16.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:41:17.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:17.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:17.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:41:17.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:41:17.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:17.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:17.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:41:17.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:17.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:17.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:17.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:17.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:17.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:17.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:17.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:41:21.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 96 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:41:21.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:21.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:21.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:41:21.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:41:21.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:41:21.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:21.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:41:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=96 2026-03-23T19:41:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:41:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:41:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:41:21.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:21.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=96 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 96 = 96 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:21.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:22.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:22.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:22.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:22.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:22.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:19 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294875,"replay_state":"idle"} 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:15 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 96 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.a20e1698-23aa-436f-b99a-a86b1f6b1a5a (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:41:22.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:41:22.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:41:22.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:22.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:41:22.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:22.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:22.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:22.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:22.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:22.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:22.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:22.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:19' 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:41:19' 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:19' 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:22.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:22.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:41:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294875,"replay_state":"idle"}2026-03-23 19:41:15' 2026-03-23T19:41:22.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:22.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:22.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:19' 2026-03-23T19:41:22.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:22.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:41:22.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:22.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:41:22.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:22.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:22.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:23.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294875,"replay_state":"idle"} 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:15 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:19' 2026-03-23T19:41:23.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:41:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:41:23.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:23.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294875,"replay_state":"idle"} 2026-03-23T19:41:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:23.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:23.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:23.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:23.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:23.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:23.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:15' 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:41:15' 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:15' 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:23.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:23.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294875,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:41:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:41:19' 2026-03-23T19:41:23.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:23.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:23.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:15' 2026-03-23T19:41:23.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:23.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:41:23.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:23.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:41:24.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:24.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:25.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:25.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:25.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:41:25.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:25.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:25.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:41:25.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:25.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:41:25.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:25.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:25.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:25.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:25.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:25.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:25.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:25.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:25.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:25.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:25.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:41:25.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:25.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:41:27.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:27.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:27.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:41:27.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:27.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:27.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:41:27.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:27.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:27.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:27.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:27.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:27.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:41:27.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:41:27.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:27.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:41:27.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:27.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:26 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:24' 2026-03-23T19:41:28.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:28.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:28.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:28.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:41:28.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:28.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:28.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:28.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:28.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:28.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:28.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:28.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:26' 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:41:26' 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:26' 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:28.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:28.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:41:26cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:41:24' 2026-03-23T19:41:28.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:28.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:26' 2026-03-23T19:41:28.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:28.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:28.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:28.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:28.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:28.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:29.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:29.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:24 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:26' 2026-03-23T19:41:29.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:29.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:29.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:29.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:41:29.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:29.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:29.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:29.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:29.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:29.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:29.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:29.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:24' 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:41:24' 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:24' 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:29.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:29.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:41:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:41:26' 2026-03-23T19:41:29.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:29.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:29.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:24' 2026-03-23T19:41:29.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:29.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:29.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:41:29.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:41:30.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:30.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:30.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:31.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:31.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:31.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:31.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:31.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:31.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:31.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:31.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:31.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:31.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:31.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:41:31.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:41:31.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:31.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:41:31.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:31.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:33.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:33.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:33.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:33.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:33.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:33.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:33.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:33.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:33.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:41:33.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:41:33.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:33.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:41:33.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:33.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:37.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:37.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:37.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:37.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:37.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:37.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:37.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:37.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:37.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:37.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:37.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:37.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:37.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:37.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:37.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:41:37.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:37.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:37.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:37.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:41:40.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 99 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:40.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:41:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:41:40.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:40.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:40.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:41:40.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:41:40.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:40.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:40.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:40.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:40.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:40.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:40.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:40.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:41:40.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:40.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:41:40.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=99 2026-03-23T19:41:40.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:41:40.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:41:40.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:41:40.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:40.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=99 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 99 = 99 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:40.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:41.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:41.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:41.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:41.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:40 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294890,"replay_state":"idle"} 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:34 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 99 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.d3bde32c-b70b-4dd2-8b66-b6ea527e8c57 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:41:41.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:41:41.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:41:41.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:41.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:41:41.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:41.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:41.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:41.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:41.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:41.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:41.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:41.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:41.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:40' 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:41:40' 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:40' 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:41.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:41.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:41:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294890,"replay_state":"idle"}2026-03-23 19:41:34' 2026-03-23T19:41:41.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:41.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:40' 2026-03-23T19:41:41.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:41.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:41:41.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:41.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:41:41.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:41.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:42.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:42.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294890,"replay_state":"idle"} 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:34 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:40' 2026-03-23T19:41:42.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:41:42.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:41:42.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:42.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294890,"replay_state":"idle"} 2026-03-23T19:41:42.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:42.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:42.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:42.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:42.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:42.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:42.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:42.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:34' 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:41:34' 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:34' 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:42.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:42.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294890,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:41:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:41:40' 2026-03-23T19:41:42.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:42.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:34' 2026-03-23T19:41:42.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:41:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:41:43.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:43.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:44.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:44.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:44.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:44.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:44.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:44.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:44.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:44.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:44.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:44.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:44.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:44.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:44.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:44.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:44.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:44.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:44.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:41:44.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:44.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:41:46.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:46.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:41:46.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:46.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:41:46.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:46.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:46.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:41:46.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:46.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:46.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:47.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:45 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:45' 2026-03-23T19:41:48.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:48.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:48.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:48.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:41:48.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:48.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:48.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:48.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:48.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:48.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:48.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:48.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:45' 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:41:45' 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:45' 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:48.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:48.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:48.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:41:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:41:45' 2026-03-23T19:41:48.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:48.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:48.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:45' 2026-03-23T19:41:48.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:48.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:48.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:48.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:48.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:49.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:45 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:45' 2026-03-23T19:41:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:41:49.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:41:49.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:49.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:41:49.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:49.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:49.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:49.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:49.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:49.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:49.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:49.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:45' 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:41:45' 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:45' 2026-03-23T19:41:49.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:49.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:49.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:49.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted45124515mirror.0vm08.local2026-03-23 19:41:45cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:41:45' 2026-03-23T19:41:49.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:49.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:45' 2026-03-23T19:41:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:41:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:41:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:41:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:41:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:41:50.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:50.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:51.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:51.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:41:51.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:51.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:51.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:41:51.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:51.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:41:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:51.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:51.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:51.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:51.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:51.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:51.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:51.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:51.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:41:51.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:51.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:41:55.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 102 2026-03-23T19:41:55.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:41:55.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:55.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:55.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:41:55.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:41:55.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:41:55.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:55.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=102 2026-03-23T19:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:41:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:41:55.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:41:55.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:41:55.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=102 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 102 = 102 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:55.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:56.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:56.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:54 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294912,"remote_snapshot_timestamp":1774294912,"replay_state":"idle"} 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:55 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 102 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.017fcd45-16cc-4e4e-8c50-78a12f16bd68 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:41:56.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:41:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:41:56.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:56.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:41:56.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:56.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:56.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:56.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:56.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:56.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:56.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:56.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:54' 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:41:54' 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:54' 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:56.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:41:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294912,"remote_snapshot_timestamp":1774294912,"replay_state":"idle"}2026-03-23 19:41:55' 2026-03-23T19:41:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:56.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:56.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:54' 2026-03-23T19:41:56.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:41:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:41:56.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:41:56.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:56.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:57.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:57.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:41:57.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294912,"remote_snapshot_timestamp":1774294912,"replay_state":"idle"} 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:55 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:54' 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:41:57.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:41:57.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:41:57.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294912,"remote_snapshot_timestamp":1774294912,"replay_state":"idle"} 2026-03-23T19:41:57.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:41:57.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:57.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:41:57.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:57.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:41:57.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:41:57.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:41:57.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:55' 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:41:55' 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:55' 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:41:57.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:41:57.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294912,"remote_snapshot_timestamp":1774294912,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:41:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:41:54' 2026-03-23T19:41:57.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:41:57.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:41:57.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:55' 2026-03-23T19:41:57.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:41:57.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:41:57.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:41:57.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:41:57.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:41:57.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:57.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:41:57.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:41:57.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:41:57.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:41:57.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:57.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:57.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:41:58.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:58.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:41:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:59.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:41:59.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:41:59.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:41:59.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:41:59.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:59.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:41:59.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:41:59.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:59.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:59.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:41:59.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:41:59.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:41:59.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:41:59.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:41:59.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:41:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:41:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:41:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:41:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:41:59.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:01.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:01.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:01.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:01.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:01.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:01.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:42:01.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:42:01.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:01.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:02.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:00 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:59' 2026-03-23T19:42:02.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:02.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:02.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:02.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:02.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:02.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:02.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:02.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:02.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:02.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:02.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:02.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:00' 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:42:00' 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:00' 2026-03-23T19:42:02.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:02.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:02.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:02.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:42:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:41:59' 2026-03-23T19:42:02.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:02.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:02.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:00' 2026-03-23T19:42:02.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:02.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:02.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:02.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:02.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:02.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:02.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:03.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:41:59 2026-03-23T19:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:00' 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:03.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:03.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:03.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:03.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:03.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:03.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:59' 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:41:59' 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:41:59' 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:03.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:03.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:41:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:42:00' 2026-03-23T19:42:03.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:03.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:03.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:41:59' 2026-03-23T19:42:03.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:03.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:03.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:03.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:03.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:42:05.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:05.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:06.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:06.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:06.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:06.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:06.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:06.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:06.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:06.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:06.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:06.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:06.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:06.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:06.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:06.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:06.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:06.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:06.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:06.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:06.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:42:10.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 105 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:42:10.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:10.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:10.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:42:10.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:42:10.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:42:10.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:10.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:42:10.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=105 2026-03-23T19:42:10.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:42:10.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:42:10.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:42:10.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:10.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:42:10.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=105 2026-03-23T19:42:10.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 105 = 105 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:10.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:11.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:10 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294924,"replay_state":"idle"} 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:04 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 105 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.250b82e3-9ca2-42a9-88b7-7c4b4b97c7f2 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:42:11.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:42:11.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:42:11.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:11.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:42:11.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:11.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:11.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:11.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:11.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:11.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:11.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:11.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:10' 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:42:10' 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:10' 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:11.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:42:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294924,"replay_state":"idle"}2026-03-23 19:42:04' 2026-03-23T19:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:11.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:10' 2026-03-23T19:42:11.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:11.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:42:11.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:11.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:11.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:12.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294924,"replay_state":"idle"} 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:04 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:10' 2026-03-23T19:42:12.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:42:12.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:42:12.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:12.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294924,"replay_state":"idle"} 2026-03-23T19:42:12.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:12.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:12.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:12.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:04' 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:42:04' 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:04' 2026-03-23T19:42:12.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:12.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294924,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:42:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:42:10' 2026-03-23T19:42:12.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:12.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:12.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:04' 2026-03-23T19:42:12.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:12.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:42:12.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:12.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:12.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:42:13.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:13.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:14.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:14.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:14.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:14.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:14.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:14.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:14.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:14.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:42:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:14.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:16.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:16.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:16.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:16.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:16.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:16.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:16.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:16.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:42:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:42:16.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:42:16.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:16.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:17.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:42:17.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:17.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:15 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:14' 2026-03-23T19:42:17.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:17.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:17.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:17.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:17.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:17.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:17.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:17.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:15' 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:42:15' 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:15' 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:42:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:42:14' 2026-03-23T19:42:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:17.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:17.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:15' 2026-03-23T19:42:17.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:17.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:17.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:17.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:17.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:17.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:18.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:18.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:14 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:15' 2026-03-23T19:42:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:18.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:18.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:18.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:14' 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:42:14' 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:14' 2026-03-23T19:42:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:18.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:18.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:18.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:42:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:42:15' 2026-03-23T19:42:18.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:18.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:18.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:14' 2026-03-23T19:42:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:18.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:18.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:18.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:18.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:18.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:18.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:42:18.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:42:20.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:20.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:21.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:21.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:21.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:21.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:21.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:21.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:21.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:21.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:21.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:21.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:21.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:21.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:21.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:21.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:21.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:42:25.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 108 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:42:25.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:25.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:42:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:42:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:25.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:25.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:42:25.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:25.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:42:25.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=108 2026-03-23T19:42:25.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:42:25.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:42:25.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:42:25.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:25.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=108 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 108 = 108 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:25.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:26.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:42:26.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:26.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:24 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294942,"remote_snapshot_timestamp":1774294942,"replay_state":"idle"} 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:25 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 108 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.ba096848-5527-43e2-b5d9-377e7eef2bf5 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:42:26.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:42:26.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:26.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:42:26.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:26.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:26.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:26.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:26.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:26.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:26.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:24' 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:42:24' 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:24' 2026-03-23T19:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:26.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:26.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:26.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:42:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294942,"remote_snapshot_timestamp":1774294942,"replay_state":"idle"}2026-03-23 19:42:25' 2026-03-23T19:42:26.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:26.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:26.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:24' 2026-03-23T19:42:26.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:42:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:26.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:42:26.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:26.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:27.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:27.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:28.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:28.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:28.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:28.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294942,"remote_snapshot_timestamp":1774294942,"replay_state":"idle"} 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:25 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:24' 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:42:28.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:28.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294942,"remote_snapshot_timestamp":1774294942,"replay_state":"idle"} 2026-03-23T19:42:28.008 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:28.009 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:28.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:28.010 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:28.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:25' 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:42:25' 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:25' 2026-03-23T19:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:28.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:28.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:28.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294942,"remote_snapshot_timestamp":1774294942,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:42:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:42:24' 2026-03-23T19:42:28.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:28.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:28.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:25' 2026-03-23T19:42:28.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:28.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:42:28.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:28.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:42:28.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:28.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:42:28.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:28.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:29.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:29.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:29.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:29.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:29.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:29.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:29.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:29.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:29.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:29.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:29.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:29.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:42:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:29.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:42:31.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:31.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:31.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:31.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:31.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:31.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:31.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:31.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:31.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:31.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:31.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:42:31.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:42:31.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:32.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:30 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:29' 2026-03-23T19:42:32.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:32.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:32.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:32.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:42:32.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:32.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:32.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:32.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:32.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:32.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:32.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:32.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:30' 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:42:30' 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:30' 2026-03-23T19:42:32.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:32.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:32.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:32.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted44024407mirror.0vm08.local2026-03-23 19:42:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:42:29' 2026-03-23T19:42:32.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:32.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:32.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:30' 2026-03-23T19:42:32.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:32.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:32.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:32.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:32.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:32.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:33.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:29 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:30' 2026-03-23T19:42:33.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:33.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:33.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:33.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:33.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:33.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:33.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:33.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:33.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:33.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:33.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:33.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:29' 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:42:29' 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:29' 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:33.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:33.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:42:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:42:30' 2026-03-23T19:42:33.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:33.877 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:33.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:29' 2026-03-23T19:42:33.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:33.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:33.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:33.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:42:33.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:42:35.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:35.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:36.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:36.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:36.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:36.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:36.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:36.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:36.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:36.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:36.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:36.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:36.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:36.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:36.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:36.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:36.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:36.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:36.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:42:39.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 111 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:39.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:42:39.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:42:39.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:42:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:42:39.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:39.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:42:39.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:39.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:42:39.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=111 2026-03-23T19:42:39.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:42:39.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:42:39.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:42:39.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=111 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 111 = 111 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:40.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:41.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:40 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294954,"replay_state":"idle"} 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:34 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 111 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.b33001c2-0d1d-4378-95d8-ca501655dda3 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:42:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:42:41.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:41.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:41.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:41.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:41.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:41.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:41.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:41.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:40' 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:42:40' 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:40' 2026-03-23T19:42:41.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:41.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:41.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:41.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:42:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294954,"replay_state":"idle"}2026-03-23 19:42:34' 2026-03-23T19:42:41.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:41.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:41.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:40' 2026-03-23T19:42:41.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:41.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:42:41.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:41.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:41.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:41.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:41.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:41.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:41.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:42.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294954,"replay_state":"idle"} 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:34 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:40' 2026-03-23T19:42:42.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:42:42.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:42:42.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:42.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294954,"replay_state":"idle"} 2026-03-23T19:42:42.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:42.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:42.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:42.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:42.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:42.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:42.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:42.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:34' 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:42:34' 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:34' 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:42.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294954,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:42:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:42:40' 2026-03-23T19:42:42.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:42.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:34' 2026-03-23T19:42:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:42.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:42:42.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:42.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:42.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:42:42.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:42:42.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:42.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:43.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:43.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:43.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:43.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:43.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:43.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:43.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:43.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:43.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:43.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:43.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:43.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:43.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:42:43.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:43.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:45.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:45.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:42:45.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:42:45.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:45.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:42:45.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:45.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:45.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:42:45.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:45.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:45.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:42:45.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:42:45.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:45.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:46.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:45 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:46.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:44' 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:46.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:47.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:47.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:47.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:47.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:47.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:47.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:47.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:47.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:47.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:45' 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:42:45' 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:45' 2026-03-23T19:42:47.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:47.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:47.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:47.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:42:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:42:44' 2026-03-23T19:42:47.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:47.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:47.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:45' 2026-03-23T19:42:47.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:47.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:47.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:48.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:48.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:48.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:48.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:48.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:48.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:48.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:48.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:48.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:44 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:45' 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:42:48.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:48.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:42:48.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:48.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:48.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:48.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:48.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:44' 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:42:44' 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:44' 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:48.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:48.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:42:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:42:45' 2026-03-23T19:42:48.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:48.155 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:48.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:44' 2026-03-23T19:42:48.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:48.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:42:48.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:42:48.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:42:48.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:42:48.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:48.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:48.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:42:48.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:48.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:48.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:48.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:42:48.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:42:49.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:49.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:50.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:50.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:50.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:50.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:50.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:50.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:50.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:50.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:50.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:50.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:50.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:50.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:50.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:42:54.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 114 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:54.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:42:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:42:54.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:54.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:54.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:42:54.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:42:54.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:42:54.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:54.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:42:54.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=114 2026-03-23T19:42:54.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:42:54.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:42:54.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:42:54.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:42:54.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:42:54.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:42:54.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=114 2026-03-23T19:42:54.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 114 = 114 2026-03-23T19:42:54.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:54.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:54.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:55.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:54 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294971,"remote_snapshot_timestamp":1774294971,"replay_state":"idle"} 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:55 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 114 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.082b224c-6a22-4b98-a918-e03ed187cfa9 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:42:56.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:42:56.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:42:56.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:56.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:42:56.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:56.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:56.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:56.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:56.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:56.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:56.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:56.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:56.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:54' 2026-03-23T19:42:56.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:42:54' 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:54' 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:56.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:56.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:42:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294971,"remote_snapshot_timestamp":1774294971,"replay_state":"idle"}2026-03-23 19:42:55' 2026-03-23T19:42:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:56.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:56.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:54' 2026-03-23T19:42:56.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:56.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:42:56.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:42:56.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:56.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:56.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:57.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294971,"remote_snapshot_timestamp":1774294971,"replay_state":"idle"} 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:55 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:54' 2026-03-23T19:42:57.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:42:57.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:42:57.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:42:57.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294971,"remote_snapshot_timestamp":1774294971,"replay_state":"idle"} 2026-03-23T19:42:57.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:42:57.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:57.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:42:57.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:57.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:42:57.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:42:57.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:42:57.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:55' 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:42:55' 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:55' 2026-03-23T19:42:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:42:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:42:57.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:42:57.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774294971,"remote_snapshot_timestamp":1774294971,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:42:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:42:54' 2026-03-23T19:42:57.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:42:57.239 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:42:57.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:55' 2026-03-23T19:42:57.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:42:57.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:42:57.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:42:57.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:42:57.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:57.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:42:57.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:57.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:42:58.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:42:58.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:42:58.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:58.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:42:58.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:42:58.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:42:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:42:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:42:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:42:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:42:58.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:42:58.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:42:58.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:58.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:58.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:42:58.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:42:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:42:58.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:42:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:42:58.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:42:58.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:42:58.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:42:58.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:42:58.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:42:58.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:00.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:00.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:43:00.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:43:00.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:00.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:43:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:00.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:00.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:43:00.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:00.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:00.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:00.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:00.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:00.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:00.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:01.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:00 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:59' 2026-03-23T19:43:01.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:01.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:01.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:01.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:01.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:01.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:01.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:01.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:00' 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:43:00' 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:00' 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:01.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:01.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:02.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:43:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:42:59' 2026-03-23T19:43:02.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:02.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:02.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:00' 2026-03-23T19:43:02.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:02.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:02.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:02.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:02.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:02.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:03.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:03.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:42:59 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:00' 2026-03-23T19:43:03.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:03.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:03.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:03.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:03.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:03.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:59' 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:42:59' 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:42:59' 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:03.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:42:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:43:00' 2026-03-23T19:43:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:03.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:42:59' 2026-03-23T19:43:03.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:03.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:43:03.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:43:04.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:05.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:05.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:05.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:05.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:05.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:05.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:05.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:05.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:05.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:05.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:05.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:05.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:05.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:05.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:05.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:43:09.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 117 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:43:09.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:09.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:09.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:43:09.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:43:09.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:09.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:09.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:09.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:43:09.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=117 2026-03-23T19:43:09.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:43:09.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:09.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:43:09.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:09.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:09.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:09.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:09.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:09.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:43:09.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:09.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=117 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 117 = 117 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:10.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:10.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:10 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294983,"replay_state":"idle"} 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:04 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 117 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.6a39c6c1-dd41-45b0-a328-9e7920eefbb0 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:43:10.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:43:10.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:43:10.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:10.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:43:10.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:10.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:10.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:10' 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:43:10' 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:10' 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:10.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:43:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294983,"replay_state":"idle"}2026-03-23 19:43:04' 2026-03-23T19:43:10.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:10.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:10.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:10' 2026-03-23T19:43:10.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:10.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:43:10.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:10.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:43:10.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:10.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:11.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:11.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294983,"replay_state":"idle"} 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:04 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:10' 2026-03-23T19:43:11.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:43:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:43:11.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:11.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294983,"replay_state":"idle"} 2026-03-23T19:43:11.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:11.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:11.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:11.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:11.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:11.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:04' 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:43:04' 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:04' 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:11.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:11.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294983,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:43:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:43:10' 2026-03-23T19:43:11.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:11.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:11.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:04' 2026-03-23T19:43:11.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:11.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:43:11.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:11.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:11.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:43:12.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:43:12.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:43:12.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:12.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:12.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:13.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:13.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:13.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:13.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:13.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:13.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:13.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:13.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:13.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:13.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:13.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:13.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:13.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:13.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:43:15.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:15.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:15.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:15.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:15.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:15.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:15.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:15.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:15.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:15.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:15.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:15.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:43:15.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:43:15.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:15.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:16.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:15 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:14' 2026-03-23T19:43:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:16.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:16.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:16.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:16.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:16.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:16.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:16.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:16.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:15' 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:43:15' 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:15' 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:16.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:16.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:43:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:43:14' 2026-03-23T19:43:16.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:16.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:16.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:15' 2026-03-23T19:43:16.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:16.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:16.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:16.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:16.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:16.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:16.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:16.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:43:16.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:16.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:17.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:14 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:15' 2026-03-23T19:43:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:17.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:17.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:17.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:17.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:17.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:17.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:17.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:17.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:17.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:17.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:17.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:14' 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:43:14' 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:14' 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:17.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:43:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:43:15' 2026-03-23T19:43:17.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:17.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:17.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:14' 2026-03-23T19:43:17.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:17.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:17.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:17.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:43:17.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:43:19.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:19.465 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:20.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:43:20.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:20.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:20.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:43:20.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:20.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:20.470 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:43:20.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:20.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:20.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:20.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:20.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:20.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:20.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:20.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:20.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:20.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:20.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:43:23.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 120 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:43:23.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:23.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:23.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:43:23.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:23.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:43:23.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:23.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:43:23.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=120 2026-03-23T19:43:23.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:43:23.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:23.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:24.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:24.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=120 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 120 = 120 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:24.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:25.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:25.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:24 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294998,"replay_state":"idle"} 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:20 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 120 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f2fdc82d-9ce4-411d-afb8-9754fb4ffc4f (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:43:25.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:43:25.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:43:25.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:25.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:43:25.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:25.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:25.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:25.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:25.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:25.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:25.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:25.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:24' 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:43:24' 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:24' 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:25.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:25.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:43:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774294998,"replay_state":"idle"}2026-03-23 19:43:20' 2026-03-23T19:43:25.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:25.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:25.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:24' 2026-03-23T19:43:25.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:25.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:43:25.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:25.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:43:25.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:25.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:25.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:26.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295000,"remote_snapshot_timestamp":1774295000,"replay_state":"idle"} 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:25 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:26.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:24' 2026-03-23T19:43:26.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:43:26.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:43:26.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:26.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295000,"remote_snapshot_timestamp":1774295000,"replay_state":"idle"} 2026-03-23T19:43:26.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:26.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:26.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:26.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:26.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:26.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:26.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:26.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:25' 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:43:25' 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:25' 2026-03-23T19:43:26.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:26.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:26.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:26.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295000,"remote_snapshot_timestamp":1774295000,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:43:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:43:24' 2026-03-23T19:43:26.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:26.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:26.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:25' 2026-03-23T19:43:26.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:26.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:43:26.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:43:26.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:26.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:43:26.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:26.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:43:27.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:43:27.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:27.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:27.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:43:27.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:27.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:28.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:28.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:28.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:28.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:43:28.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:28.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:30.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:30.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:30.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:43:30.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:30.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:30.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:43:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:30.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:43:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:43:30.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:30.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:31.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:30 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:29' 2026-03-23T19:43:31.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:31.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:31.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:31.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:31.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:31.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:31.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:31.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:31.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:31.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:31.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:31.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:31.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:30' 2026-03-23T19:43:31.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:43:30' 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:30' 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:31.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:31.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:43:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:43:29' 2026-03-23T19:43:31.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:31.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:31.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:30' 2026-03-23T19:43:31.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:31.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:31.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:31.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:32.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:32.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:29 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:30' 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:32.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:32.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:32.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:32.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:32.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:32.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:29' 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:43:29' 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:29' 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:32.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:32.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:43:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:43:30' 2026-03-23T19:43:32.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:32.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:32.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:29' 2026-03-23T19:43:32.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:32.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:32.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:32.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:43:32.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:43:33.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:33.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:34.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:34.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:34.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:34.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:34.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:34.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:34.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:34.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:34.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:43:34.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:43:34.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:34.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:43:34.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:34.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:36.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:36.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:36.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:36.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:36.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:36.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:36.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:36.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:36.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:36.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:36.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:36.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:36.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:36.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:43:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 123 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:43:40.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:40.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:40.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:43:40.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:43:40.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:40.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:43:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=123 2026-03-23T19:43:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:43:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:40.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:43:40.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:40.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:43:40.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:40.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:40.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=123 2026-03-23T19:43:40.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 123 = 123 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:40.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:41.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:40 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295012,"replay_state":"idle"} 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:34 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 123 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.58db701b-43a0-48b3-bf65-73c947dc9d8c (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:41.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:43:41.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:41.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:41.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:41.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:41.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:41.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:41.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:41.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:40' 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:43:40' 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:40' 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:41.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:41.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:43:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295012,"replay_state":"idle"}2026-03-23 19:43:34' 2026-03-23T19:43:41.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:41.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:41.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:40' 2026-03-23T19:43:41.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:41.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:43:41.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:41.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:41.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:42.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295012,"replay_state":"idle"} 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:34 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:40' 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:43:42.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:43:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:42.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295012,"replay_state":"idle"} 2026-03-23T19:43:42.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:42.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:42.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:42.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:42.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:42.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:42.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:42.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:42.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:34' 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:43:34' 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:34' 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:42.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:42.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295012,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:43:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:43:40' 2026-03-23T19:43:42.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:42.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:42.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:34' 2026-03-23T19:43:42.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:42.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:43:42.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:42.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:43:42.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:42.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:43:43.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:44.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:44.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:44.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:44.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:44.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:44.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:44.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:44.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:44.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:44.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:44.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:43:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:44.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:46.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:43:46.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:43:46.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:46.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:43:46.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:46.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:46.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:43:46.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:46.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:46.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:43:46.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:43:46.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:46.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:45 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:45' 2026-03-23T19:43:47.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:47.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:47.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:47.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:43:47.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:47.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:47.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:47.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:45' 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:43:45' 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:45' 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:43:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image demoted2026-03-23 19:43:45' 2026-03-23T19:43:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:47.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:45' 2026-03-23T19:43:47.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:47.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:47.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:48.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:48.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:48.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:48.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:45 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:45' 2026-03-23T19:43:48.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:43:48.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:43:48.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:43:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:48.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:48.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:48.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:48.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:48.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:48.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:45' 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:43:45' 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:45' 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:48.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:48.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image demoted45124515mirror.0vm08.local2026-03-23 19:43:45cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:43:45' 2026-03-23T19:43:48.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:48.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:48.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:45' 2026-03-23T19:43:48.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:43:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:43:48.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:43:48.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:43:49.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:49.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:43:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:50.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:43:50.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:43:50.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:43:50.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:43:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:43:50.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:43:50.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:43:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:43:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:43:51.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 126 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:43:51.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:43:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:43:51.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:51.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:43:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:43:51.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:51.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:43:51.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=126 2026-03-23T19:43:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:43:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:51.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:51.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:51.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:51.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=125 2026-03-23T19:43:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 126 = 125 2026-03-23T19:43:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:51.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:51.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:51.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:51.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:51.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=125 2026-03-23T19:43:51.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 126 = 125 2026-03-23T19:43:51.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:43:51.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:43:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:52.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:43:52.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:43:52.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:43:52.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=126 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 126 = 126 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:52.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:53.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:53.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:53.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:49 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295029,"replay_state":"idle"} 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:50 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 121 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.80e0356b-a505-4bf1-a853-a25e0f897238 (demoted peer_uuids:[]) 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 125 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.457d4133-3265-483d-babd-2c05573b2b48 (peer_uuids:[]) 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 126 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.21150f46-db3e-412a-a764-41b7199a128f (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:43:53.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:43:53.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:43:53.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:43:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:53.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:53.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:53.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:53.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:53.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:53.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:49' 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:43:49' 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:49' 2026-03-23T19:43:53.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:53.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:53.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:53.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:43:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295029,"replay_state":"idle"}2026-03-23 19:43:50' 2026-03-23T19:43:53.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:53.812 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:53.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:49' 2026-03-23T19:43:53.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:53.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:43:53.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:43:53.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:53.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:54.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295029,"replay_state":"idle"} 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:50 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:54' 2026-03-23T19:43:54.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:43:54.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:43:54.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:43:54.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295029,"replay_state":"idle"} 2026-03-23T19:43:54.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:43:54.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:54.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:43:54.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:54.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:43:54.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:43:54.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:43:54.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:50' 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:43:50' 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:50' 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:43:54.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:43:54.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295029,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:43:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:43:54' 2026-03-23T19:43:54.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:43:54.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:43:54.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:50' 2026-03-23T19:43:54.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:43:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:43:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:43:54.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:43:59.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:43:59.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:00.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:00.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:00.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:00.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:00.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:00.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:00.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:00.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:00.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:00.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:00.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:44:00.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:44:00.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:00.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:01.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:00 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:59' 2026-03-23T19:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:01.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:01.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:01.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:01.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:01.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:01.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:01.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:01.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:01.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:00' 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:44:00' 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:00' 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:01.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:01.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:44:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:43:59' 2026-03-23T19:44:01.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:01.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:01.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:00' 2026-03-23T19:44:01.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:01.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:01.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:01.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:01.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:02.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:43:59 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:02.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:02.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:00' 2026-03-23T19:44:02.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:02.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:02.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:02.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:02.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:02.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:02.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:02.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:02.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:02.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:02.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:02.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:02.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:59' 2026-03-23T19:44:02.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:43:59' 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:43:59' 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:02.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:02.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:43:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:44:00' 2026-03-23T19:44:02.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:02.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:02.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:43:59' 2026-03-23T19:44:02.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:44:02.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:44:04.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:44:04.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:44:04.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:04.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:04.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:05.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:05.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:05.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:05.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:05.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:05.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:05.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:05.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:05.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:05.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:05.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:44:09.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 129 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:44:09.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:09.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:09.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:44:09.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:44:09.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:44:09.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:09.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:44:09.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=129 2026-03-23T19:44:09.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:44:09.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:44:09.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:44:09.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:09.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:09.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:09.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:09.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:09.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:44:09.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:09.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=129 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 129 = 129 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:09.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:10.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:10.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:10.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:10 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295043,"replay_state":"idle"} 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:04 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 129 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f34e25a1-520c-4a31-a99a-2f1ad5095ff2 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:44:10.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:44:10.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:44:10.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:10.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:44:10.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:10.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:10.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:10.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:10.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:10.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:10' 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:44:10' 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:10' 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:10.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:44:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295043,"replay_state":"idle"}2026-03-23 19:44:04' 2026-03-23T19:44:10.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:10.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:10.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:10' 2026-03-23T19:44:10.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:10.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:44:10.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:10.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:44:10.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:10.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:10.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:10.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:11.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:12.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295043,"replay_state":"idle"} 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:04 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:12.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:12.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:10' 2026-03-23T19:44:12.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:44:12.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:44:12.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:12.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295043,"replay_state":"idle"} 2026-03-23T19:44:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:12.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:12.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:12.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:12.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:12.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:04' 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:44:04' 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:04' 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:12.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295043,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:44:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:44:10' 2026-03-23T19:44:12.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:12.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:12.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:04' 2026-03-23T19:44:12.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:12.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:44:12.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:12.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:12.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:44:12.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:12.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:13.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:13.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:13.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:13.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:13.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:13.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:13.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:13.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:13.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:13.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:13.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:13.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:13.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:13.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:13.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:13.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:13.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:44:13.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:13.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:15.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:15.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:15.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:15.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:15.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:15.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:15.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:15.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:44:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:44:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:16.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:15 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:14' 2026-03-23T19:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:16.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:16.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:16.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:16.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:16.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:16.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:15' 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:44:15' 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:15' 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:16.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:44:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:44:14' 2026-03-23T19:44:16.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:16.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:16.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:15' 2026-03-23T19:44:16.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:16.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:16.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:16.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:16.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:17.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:14 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:15' 2026-03-23T19:44:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:17.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:17.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:17.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:17.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:17.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:17.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:14' 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:44:14' 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:14' 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:17.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:17.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:44:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:44:15' 2026-03-23T19:44:17.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:17.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:17.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:14' 2026-03-23T19:44:17.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:17.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:17.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:44:17.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:44:18.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:18.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:19.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:19.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:19.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:19.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:19.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:19.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:19.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:19.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:19.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:19.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:19.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:44:19.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:44:19.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:19.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:44:19.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:19.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:44:20.762 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:44:20.759+0000 7f9862b48640 -1 reset not still connected to 0x55d6bc46dee0 2026-03-23T19:44:20.762 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:44:20.759+0000 7f9862b48640 -1 reset not still connected to 0x55d6bd4b1860 2026-03-23T19:44:20.762 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:44:20.759+0000 7f9862b48640 -1 reset not still connected to 0x55d6beb10410 2026-03-23T19:44:20.762 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:44:20.759+0000 7f9862b48640 -1 reset not still connected to 0x55d6beb11380 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:21.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:21.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:21.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:21.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:21.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:21.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:21.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:21.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:21.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:21.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:21.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:21.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:21.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:44:25.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 132 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:44:25.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:44:25.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:44:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:44:25.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:44:25.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:25.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:44:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=132 2026-03-23T19:44:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:44:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:44:25.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:44:25.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:25.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:25.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:25.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:44:25.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:25.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:44:25.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=132 2026-03-23T19:44:25.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 132 = 132 2026-03-23T19:44:25.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:25.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:24 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295062,"remote_snapshot_timestamp":1774295062,"replay_state":"idle"} 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:25 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 132 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.8d93bd9e-7c5f-4a72-b8f5-77bfb20f6d7e (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:44:26.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:44:26.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:44:26.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:26.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:44:26.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:26.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:26.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:26.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:26.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:26.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:26.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:26.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:26.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:24' 2026-03-23T19:44:26.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:44:24' 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:24' 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:26.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:26.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:44:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295062,"remote_snapshot_timestamp":1774295062,"replay_state":"idle"}2026-03-23 19:44:25' 2026-03-23T19:44:26.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:26.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:26.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:24' 2026-03-23T19:44:26.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:44:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:26.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:26.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:27.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:27.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:27.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:27.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:27.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295062,"remote_snapshot_timestamp":1774295062,"replay_state":"idle"} 2026-03-23T19:44:27.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:25 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:24' 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:44:27.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:27.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295062,"remote_snapshot_timestamp":1774295062,"replay_state":"idle"} 2026-03-23T19:44:27.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:27.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:27.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:27.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:27.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:27.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:27.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:27.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:25' 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:44:25' 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:25' 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:27.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295062,"remote_snapshot_timestamp":1774295062,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:44:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:44:24' 2026-03-23T19:44:27.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:27.770 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:27.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:25' 2026-03-23T19:44:27.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:27.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:44:27.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:27.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:44:27.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:44:28.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:44:28.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:44:28.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:28.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:28.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:29.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:29.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:29.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:29.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:29.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:29.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:29.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:29.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:29.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:29.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:29.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:29.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:29.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:29.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:29.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:29.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:29.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:29.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:29.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:29.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:29.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:44:29.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:29.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:44:31.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:31.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:31.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:31.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:31.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:31.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:31.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:31.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:31.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:31.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:31.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:44:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:44:31.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:31.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:44:31.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:31.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:30 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:29' 2026-03-23T19:44:32.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:32.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:32.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:32.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:32.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:32.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:32.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:32.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:32.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:32.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:32.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:32.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:30' 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:44:30' 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:30' 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:32.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:32.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:44:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:44:29' 2026-03-23T19:44:32.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:32.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:30' 2026-03-23T19:44:32.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:32.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:32.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:32.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:32.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:33.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:29 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:30' 2026-03-23T19:44:33.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:33.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:33.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:33.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:33.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:33.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:33.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:33.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:33.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:33.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:33.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:33.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:29' 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:44:29' 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:33.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:33.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:29' 2026-03-23T19:44:33.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:33.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:33.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:33.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:44:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:44:30' 2026-03-23T19:44:33.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:33.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:33.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:29' 2026-03-23T19:44:33.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:33.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:44:33.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:44:34.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:34.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:35.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:35.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:35.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:35.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:35.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:35.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:35.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:35.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:35.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:35.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:35.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:35.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:35.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:44:39.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 135 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:44:39.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:39.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:39.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:44:39.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:44:39.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:39.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:44:39.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:39.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:44:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=135 2026-03-23T19:44:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:44:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:44:39.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:44:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:44:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:40.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=135 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 135 = 135 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:41.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:40 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:41.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295073,"replay_state":"idle"} 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:34 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 135 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.cd6db537-9a8b-402b-be94-3ec7e18426b3 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:44:41.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:41.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:44:41.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:41.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:41.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:41.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:41.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:41.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:41.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:41.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:40' 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:44:40' 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:40' 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:41.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:41.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:44:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295073,"replay_state":"idle"}2026-03-23 19:44:34' 2026-03-23T19:44:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:40' 2026-03-23T19:44:41.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:41.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:44:41.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:41.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:44:41.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:41.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:41.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:42.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295073,"replay_state":"idle"} 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:34 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:40' 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:44:42.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:44:42.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:42.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295073,"replay_state":"idle"} 2026-03-23T19:44:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:42.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:42.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:42.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:42.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:42.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:34' 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:44:34' 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:34' 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:42.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:42.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295073,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:44:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:44:40' 2026-03-23T19:44:42.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:42.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:34' 2026-03-23T19:44:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:42.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:44:42.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:42.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:42.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:42.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:43.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:43.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:43.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:43.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:43.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:43.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:43.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:43.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:43.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:43.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:43.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:43.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:43.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:43.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:44:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:45.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:45.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:44:45.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:45.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:45.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:44:45.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:45.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:45.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:44:45.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:46.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:46.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:44:46.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:44:46.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:46.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:47.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:47.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:45 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:44' 2026-03-23T19:44:47.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:47.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:47.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:47.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:47.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:47.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:47.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:47.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:47.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:47.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:47.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:47.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:45' 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:44:45' 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:45' 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:47.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:44:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:44:44' 2026-03-23T19:44:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:45' 2026-03-23T19:44:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:47.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:47.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:47.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:47.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:47.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:48.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:44 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:45' 2026-03-23T19:44:48.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:44:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:44:48.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:44:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:48.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:48.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:48.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:48.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:48.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:48.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:44' 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:44:44' 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:44' 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:48.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:48.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:44:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:44:45' 2026-03-23T19:44:48.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:48.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:48.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:44' 2026-03-23T19:44:48.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:48.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:44:48.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:44:48.643 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:44:48.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:44:50.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:50.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:51.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:51.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:51.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:51.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:51.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:51.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:51.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:44:51.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:51.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:44:55.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 138 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:44:55.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:44:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:55.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:44:55.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:44:55.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:44:55.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:55.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:44:55.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=138 2026-03-23T19:44:55.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:44:55.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:44:55.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:44:55.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:44:55.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:44:55.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=138 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 138 = 138 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:55.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:56.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:44:56.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:56.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:56.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:54 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295092,"remote_snapshot_timestamp":1774295092,"replay_state":"idle"} 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:55 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 138 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.f59fb3c3-fffc-48c0-bbf1-6ac3b4a6dd80 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:44:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:44:57.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:44:57.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:44:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:57.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:57.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:57.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:57.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:57.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:57.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:57.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:54' 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:44:54' 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:54' 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:57.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:57.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:44:54cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295092,"remote_snapshot_timestamp":1774295092,"replay_state":"idle"}2026-03-23 19:44:55' 2026-03-23T19:44:57.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:57.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:54' 2026-03-23T19:44:57.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:57.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:44:57.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:44:57.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:44:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:44:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:44:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:57.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:58.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295092,"remote_snapshot_timestamp":1774295092,"replay_state":"idle"} 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:55 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:54' 2026-03-23T19:44:58.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:44:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:44:58.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:44:58.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295092,"remote_snapshot_timestamp":1774295092,"replay_state":"idle"} 2026-03-23T19:44:58.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:44:58.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:58.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:44:58.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:58.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:44:58.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:44:58.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:44:58.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:44:58.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:55' 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:44:55' 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:55' 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:44:58.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:44:58.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295092,"remote_snapshot_timestamp":1774295092,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:44:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:44:54' 2026-03-23T19:44:58.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:44:58.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:44:58.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:55' 2026-03-23T19:44:58.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:44:58.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:44:58.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:44:58.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:58.353 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:44:58.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:58.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:44:59.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:44:59.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:59.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:44:59.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:44:59.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:44:59.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:44:59.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:44:59.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:59.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:59.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:44:59.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:44:59.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:44:59.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:44:59.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:44:59.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:44:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:44:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:44:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:44:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:44:59.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:01.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:01.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:45:01.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:01.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:01.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:45:01.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:45:01.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:01.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:01.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:45:01.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:01.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:01.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:01.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:01.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:01.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:01.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:01.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:01.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:45:01.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:01.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:02.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:00 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:59' 2026-03-23T19:45:02.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:02.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:02.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:02.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:02.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:02.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:02.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:02.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:02.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:02.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:02.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:02.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:00' 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:45:00' 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:00' 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:02.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:02.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:45:00cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:44:59' 2026-03-23T19:45:02.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:02.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:02.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:00' 2026-03-23T19:45:02.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:02.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:02.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:02.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:02.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:02.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:03.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:44:59 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:00' 2026-03-23T19:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:04.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:04.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:04.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:04.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:04.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:04.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:04.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:04.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:04.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:04.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:04.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:59' 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:44:59' 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:04.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:44:59' 2026-03-23T19:45:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:04.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:04.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:44:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:45:00' 2026-03-23T19:45:04.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:04.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:04.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:44:59' 2026-03-23T19:45:04.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:04.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:04.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:04.073 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:45:04.074 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:45:05.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:45:05.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:05.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:06.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:06.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:06.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:06.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:06.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:06.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:06.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:06.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:06.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:06.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:06.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:06.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:06.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:06.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:06.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:45:06.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:06.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:08.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:08.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:08.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:08.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:08.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:08.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:08.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:08.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:08.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:08.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:08.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:08.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:08.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:08.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:45:08.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:08.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:12.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:12.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:12.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:12.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:12.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:12.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:12.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:12.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:12.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:12.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:12.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:12.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:12.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:45:12.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:12.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:12.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:12.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:45:15.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 141 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:15.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:15.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:45:15.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:45:15.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:15.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:15.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:45:15.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:45:15.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:45:15.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:15.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:45:15.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=141 2026-03-23T19:45:15.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:45:15.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:45:15.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:45:15.632 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:15.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:45:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=141 2026-03-23T19:45:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 141 = 141 2026-03-23T19:45:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:45:15.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:15.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:16.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:16.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:16.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:16.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:15 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295105,"replay_state":"idle"} 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:09 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 141 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.0f4effdc-7768-4698-ae82-c6ed6f115bbf (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:45:16.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:45:16.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:45:16.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:45:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:16.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:16.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:16.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:16.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:16.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:15' 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:45:15' 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:15' 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:16.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:16.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:45:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295105,"replay_state":"idle"}2026-03-23 19:45:09' 2026-03-23T19:45:16.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:16.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:16.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:15' 2026-03-23T19:45:16.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:16.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:45:16.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:16.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:45:16.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:16.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295105,"replay_state":"idle"} 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:09 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:17.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:17.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:15' 2026-03-23T19:45:17.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:45:17.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:45:17.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295105,"replay_state":"idle"} 2026-03-23T19:45:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:17.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:17.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:17.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:17.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:17.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:17.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:09' 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:45:09' 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:09' 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:17.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:17.805 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:17.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295105,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:45:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:45:15' 2026-03-23T19:45:17.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:17.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:17.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:09' 2026-03-23T19:45:17.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:17.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:45:17.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:17.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:17.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:45:18.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:19.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:19.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:19.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:19.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:19.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:19.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:19.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:19.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:19.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:19.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:19.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:19.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:45:19.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:19.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:21.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:21.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:21.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:21.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:21.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:21.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:21.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:21.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:21.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:21.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:21.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:21.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:21.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:21.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:21.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:22.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:20 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:19' 2026-03-23T19:45:22.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:22.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:22.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:22.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:22.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:22.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:22.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:22.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:20' 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:45:20' 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:20' 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:22.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:45:20cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:45:19' 2026-03-23T19:45:22.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:22.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:22.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:20' 2026-03-23T19:45:22.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:22.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:22.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:22.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:22.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:23.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:19 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:23.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:20' 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:23.382 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:23.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:23.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:23.383 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:23.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:23.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:23.384 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:23.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:23.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:23.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:19' 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:45:19' 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:19' 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:23.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:23.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:45:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:45:20' 2026-03-23T19:45:23.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:23.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:23.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:19' 2026-03-23T19:45:23.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:23.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:23.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:23.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:23.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:45:23.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:45:24.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:45:24.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:45:24.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:24.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:24.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:24.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:24.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:25.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:25.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:25.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:45:25.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:45:25.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:25.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:25.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:45:25.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:25.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:25.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:25.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:25.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:25.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:25.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:45:29.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 144 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:45:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:45:29.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:29.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:45:29.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:29.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:45:29.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=144 2026-03-23T19:45:29.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:45:29.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:45:29.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:45:29.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:29.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=144 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 144 = 144 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:30.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:29 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:30.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295123,"replay_state":"idle"} 2026-03-23T19:45:30.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:25 2026-03-23T19:45:30.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:45:30.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 144 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.da5a4ec1-4d3e-4ead-950f-512bbb53d42e (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:45:30.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:45:30.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:45:30.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:30.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:45:30.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:30.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:30.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:30.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:30.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:30.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:30.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:30.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:30.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:29' 2026-03-23T19:45:30.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:45:29' 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:29' 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:30.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:45:29cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295123,"replay_state":"idle"}2026-03-23 19:45:25' 2026-03-23T19:45:30.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:30.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:29' 2026-03-23T19:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:30.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:31.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:45:31.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:31.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:31.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295123,"replay_state":"idle"} 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:25 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:29' 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:45:31.928 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:45:31.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:31.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295123,"replay_state":"idle"} 2026-03-23T19:45:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:31.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:31.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:31.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:25' 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:45:25' 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:25' 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:31.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295123,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:45:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:45:29' 2026-03-23T19:45:31.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:31.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:31.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:25' 2026-03-23T19:45:31.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:31.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:45:31.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:31.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:31.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-03-23T19:45:32.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:45:32.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:32.856 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:33.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:33.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:45:33.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:33.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:33.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:45:33.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:33.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:33.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:33.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:33.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:33.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:33.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:33.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:33.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:33.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:33.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = started 2026-03-23T19:45:33.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:33.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:35.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:35.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:45:35.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:35.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:35.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:45:35.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:35.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:35.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:35.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:36.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:36.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:36.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:36.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:45:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:45:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:36.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:36.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:37.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:35 2026-03-23T19:45:37.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:34' 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:37.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:37.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:37.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:37.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:37.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:37.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:37.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:37.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:37.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:37.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:35' 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:45:35' 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:35' 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:37.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:37.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:45:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:45:34' 2026-03-23T19:45:37.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:37.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:37.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:35' 2026-03-23T19:45:37.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:37.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:37.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:37.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:37.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:38.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:38.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:34 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:35' 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:38.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:38.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:38.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:38.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:38.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:38.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:38.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:34' 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:45:34' 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:34' 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:38.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:38.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:45:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:45:35' 2026-03-23T19:45:38.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:38.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:38.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:34' 2026-03-23T19:45:38.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:38.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:38.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:38.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:45:38.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-03-23T19:45:40.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:40.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:41.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test started 2026-03-23T19:45:41.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:41.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:41.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:41.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:41.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:41.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:41.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:41.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:41.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:41.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:41.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:41.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:41.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-03-23T19:45:44.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 147 2026-03-23T19:45:44.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster2 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:45:44.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:44.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:45:44.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:44.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:45:44.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:45:44.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:44.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:44.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:45:44.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:44.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:45:44.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=147 2026-03-23T19:45:44.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:45:44.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:45:44.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:45:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:44.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=147 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 147 = 147 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:45.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:45.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:40 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295138,"replay_state":"idle"} 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:39 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 146 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.c54b18df-7cea-4faf-902b-ed6276b9c656 (peer_uuids:[]) 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 147 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.06a28bb1-def4-4057-9843-a112ef0e3733 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:45:45.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:45:45.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:45:45.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:45.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:45:45.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:45.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:45.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:45.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:45.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:45.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:45.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:45.421 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:40' 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-03-23 19:45:40' 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:40' 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:45.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:45.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary44024407mirror.0vm08.local2026-03-23 19:45:45cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295138,"replay_state":"idle"}2026-03-23 19:45:39' 2026-03-23T19:45:45.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:45.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:45.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:45' 2026-03-23T19:45:45.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:45.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:45:45.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:45:45.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:45:45.670 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:45.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:46.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:46.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295138,"replay_state":"idle"} 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:39 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:45' 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:45:46.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:45:46.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:46.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295138,"replay_state":"idle"} 2026-03-23T19:45:46.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:46.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:46.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:46.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:46.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:39' 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-03-23 19:45:39' 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:39' 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:46.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:46.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295138,"replay_state":"idle"}45124515mirror.0vm08.local2026-03-23 19:45:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:45:45' 2026-03-23T19:45:46.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:46.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:46.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:39' 2026-03-23T19:45:46.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:46.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:45:46.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:45:46.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:45:46.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror test 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:46.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-03-23T19:45:48.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:48.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:49.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:45:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:49.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:49.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:45:49.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:49.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:49.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:45:49.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:49.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:45:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:45:49.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:45:49.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:49.961 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:45:49.959+0000 7fedd70f0640 -1 reset not still connected to 0x56341687b1e0 2026-03-23T19:45:49.961 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:45:49.959+0000 7fedd70f0640 -1 reset not still connected to 0x563416c17380 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:50.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:50.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:45 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:49' 2026-03-23T19:45:50.317 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:50.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:45:50.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:50.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:52.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:52.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:50 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:52.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:49' 2026-03-23T19:45:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:52.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:52.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:52.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:52.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:52.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:52.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:52.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:52.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:52.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:52.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:50' 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-03-23 19:45:50' 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:50' 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:52.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:52.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary44024407mirror.0vm08.local2026-03-23 19:45:50cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:45:49' 2026-03-23T19:45:52.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:52.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:52.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:50' 2026-03-23T19:45:52.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:52.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:52.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:52.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:52.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:52.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:53.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-03-23T19:45:53.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:53.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:53.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:53.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:53.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:53.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:53.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:53.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:53.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:49 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:50' 2026-03-23T19:45:53.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:45:53.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:45:53.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:45:53.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:45:53.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:45:53.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:53.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:45:53.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:53.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:45:53.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:45:53.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:45:53.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:49' 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-03-23 19:45:49' 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:49' 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:45:53.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:45:53.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+unknownremote image is not primary45124515mirror.0vm08.local2026-03-23 19:45:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:45:50' 2026-03-23T19:45:53.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:45:53.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:45:53.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:49' 2026-03-23T19:45:53.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:45:53.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:45:53.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:45:53.549 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:45:53.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster2 mirror test 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:45:53.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-03-23T19:45:55.316 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:55.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:45:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:45:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:55.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:56.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:45:56.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:45:56.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:45:56.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:45:56.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:45:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:45:56.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:45:56.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:45:56.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:45:56.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:45:56.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:45:56.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-03-23T19:45:59.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 150 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.status 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.status 2026-03-23T19:45:59.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:59.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:59.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.status 2026-03-23T19:45:59.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.status 2026-03-23T19:45:59.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.status 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-03-23T19:45:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:59.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:45:59.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=150 2026-03-23T19:45:59.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:45:59.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:45:59.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.status 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-03-23T19:45:59.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:45:59.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=150 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 150 = 150 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:45:59.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:45:59.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:00.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-03-23T19:46:00.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:59 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"} 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:55 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 150 .mirror.primary.4e71904d-0696-4b3f-a95f-dcae5da3a2cf.7cdfa277-d0cc-46d0-b39a-37af2da82675 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:46:00.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:46:00.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:00.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:00.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:00.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:00.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:46:00.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:00.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:59' 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-03-23 19:45:59' 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:59' 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:00.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:45:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}2026-03-23 19:45:55' 2026-03-23T19:46:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:00.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:59' 2026-03-23T19:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:46:00.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:46:00.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:00.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:46:00.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:00.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:01.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4e71904d-0696-4b3f-a95f-dcae5da3a2cf 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"} 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:55 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:45:59' 2026-03-23T19:46:01.942 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:46:01.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:46:01.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:01.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"} 2026-03-23T19:46:01.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:01.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:01.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:01.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:01.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:01.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:46:01.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:01.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:55' 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:45:55' 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:45:55' 2026-03-23T19:46:01.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:01.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:01.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:01.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:45:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:45:59' 2026-03-23T19:46:01.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:01.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:46:01.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:45:55' 2026-03-23T19:46:01.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:46:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:46:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:01.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:46:01.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:01.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:01.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:01.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_mirror_snaps cluster1 mirror test 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 snap ls mirror/test --all 2026-03-23T19:46:02.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c -F ' mirror (' 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 3 -le 3 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_mirror_snaps cluster2 mirror test 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:46:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 snap ls mirror/test --all 2026-03-23T19:46:02.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c -F ' mirror (' 2026-03-23T19:46:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 -le 3 2026-03-23T19:46:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: force promote' 2026-03-23T19:46:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:02.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:02 TEST: force promote 2026-03-23T19:46:02.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:02 TEST: force promote 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ force_promote_image=test_force_promote 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote snapshot 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_force_promote 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:46:02.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:02.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_force_promote 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_force_promote snapshot 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:02.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote snapshot 2026-03-23T19:46:02.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:02.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_force_promote': 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15111ee3c32 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15111ee3c32 2026-03-23T19:46:02.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:02 2026 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:02 2026 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:02 2026 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 5afacd82-27c0-4e0f-8fd1-299b98288e55 2026-03-23T19:46:02.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:46:02.503 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:46:02.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:46:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:46:02.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2439.02 bytes/sec: 9.5 MiB/s 2026-03-23T19:46:02.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:02.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:46:03.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:46:03.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:03.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:03.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:03.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:46:03.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:03.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:03.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:46:03.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:03.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:03.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:03.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:46:03.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:03.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:46:03.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:03.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:03.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:03.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:03.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:03.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:03.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:46:03.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:46:03.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:46:03.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:46:03.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:03.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:03.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:46:03.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:46:03.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:46:03.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:46:03.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:46:03.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:46:03.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:03.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:03.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:03.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:03.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:03.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:46:04.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:46:04.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:04.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:04.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:04.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:46:04.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:04.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:04.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:04.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:04.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:04.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:04.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:46:04.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:04.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:04.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:04.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:05.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:05.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:46:05.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:46:05.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote started 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:46:05.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:05.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:05.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_force_promote started 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:06.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:06.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:06.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:06.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:06.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:06.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:06.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:06.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:06.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:06.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:06.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:06.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:06.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:06.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:06.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:46:06.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:46:06.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_force_promote 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote 2026-03-23T19:46:06.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 152 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_force_promote 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_force_promote.status 2026-03-23T19:46:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_force_promote.status 2026-03-23T19:46:06.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote.status 2026-03-23T19:46:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_force_promote.status 2026-03-23T19:46:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_force_promote.status 2026-03-23T19:46:06.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote.status 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote.status 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote --format xml 2026-03-23T19:46:06.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:06.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:46:06.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=152 2026-03-23T19:46:06.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:46:06.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:06.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-03-23T19:46:06.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:06.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:06.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=151 2026-03-23T19:46:06.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 152 = 151 2026-03-23T19:46:06.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:06.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-03-23T19:46:07.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:07.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=151 2026-03-23T19:46:07.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 152 = 151 2026-03-23T19:46:07.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:07.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote.status 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-03-23T19:46:08.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:08.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=152 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 152 = 152 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+replaying 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:08.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+replaying '' '' 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:09.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:09.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:09.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote: 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 5afacd82-27c0-4e0f-8fd1-299b98288e55 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295162,"replay_state":"idle"} 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:04 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:04' 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:46:09.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:09.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295162,"replay_state":"idle"} 2026-03-23T19:46:09.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:09.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:09.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:09.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:09.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:09.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:46:09.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:09.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:04' 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+replaying '2026-03-23 19:46:04' 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:09.236 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:04' 2026-03-23T19:46:09.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:09.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:09.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:09.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:45:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:04test_force_promote5afacd82-27c0-4e0f-8fd1-299b98288e55up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295162,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:46:04cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:04' 2026-03-23T19:46:09.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:09.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-03-23T19:46:09.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:04' 2026-03-23T19:46:09.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-03-23T19:46:09.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:46:09.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:09.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:46:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-03-23T19:46:09.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:10.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:10.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote: 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 5afacd82-27c0-4e0f-8fd1-299b98288e55 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:09 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295162,"replay_state":"idle"} 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:04 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 151 .mirror.primary.5afacd82-27c0-4e0f-8fd1-299b98288e55.8acd0166-0e33-4075-92e1-eaadc0687152 (peer_uuids:[]) 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 152 .mirror.primary.5afacd82-27c0-4e0f-8fd1-299b98288e55.66259543-e6a7-496c-980d-01258f005f8c (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:46:10.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:46:10.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:46:10.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:10.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:46:10.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:10.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:10.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:10.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:10.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:10.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:46:10.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:10.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:09' 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-03-23 19:46:09' 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:09' 2026-03-23T19:46:10.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:10.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:10.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:10.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:46:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}2026-03-23 19:45:55test_force_promote5afacd82-27c0-4e0f-8fd1-299b98288e55up+stoppedlocal image is primary45194524mirror.1vm08.local2026-03-23 19:46:09cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295162,"replay_state":"idle"}2026-03-23 19:46:04' 2026-03-23T19:46:10.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:10.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-03-23T19:46:10.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:09' 2026-03-23T19:46:10.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-03-23T19:46:10.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:46:10.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:10.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_force_promote --force 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force=--force 2026-03-23T19:46:10.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote --force 2026-03-23T19:46:15.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote stopped 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:15.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_force_promote stopped 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:16.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:16.432 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:16.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:16.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:16.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:16.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:16.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:16.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:16.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:16.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:46:16.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:46:16.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:17.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:17.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:46:17.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:17.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:17.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:46:17.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:17.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:17.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:46:17.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:17.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-03-23T19:46:17.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:17.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:17.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:17.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:46:17.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:17.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:17.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:46:17.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-03-23T19:46:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:46:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:46:17.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+stopped 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:17.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:18.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+stopped '' '' 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:18.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote: 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 5afacd82-27c0-4e0f-8fd1-299b98288e55 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:15 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:14 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 154 .mirror.primary.5afacd82-27c0-4e0f-8fd1-299b98288e55.b824796e-e3c5-4b9f-946c-67ef91736e83 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:46:18.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:46:18.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:46:18.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:18.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:46:18.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:18.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:18.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:18.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:18.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:18.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:46:18.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:18.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:15' 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+stopped '2026-03-23 19:46:15' 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:15' 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:18.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:18.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK1144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:45:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:14test_force_promote5afacd82-27c0-4e0f-8fd1-299b98288e55up+stoppedlocal image is primary44094414mirror.1vm08.local2026-03-23 19:46:15cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:14' 2026-03-23T19:46:18.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:18.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-03-23T19:46:18.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:15' 2026-03-23T19:46:18.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-03-23T19:46:18.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:46:18.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:18.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:18.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:19.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:19.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:19.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote: 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 5afacd82-27c0-4e0f-8fd1-299b98288e55 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:14 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:15 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 151 .mirror.primary.5afacd82-27c0-4e0f-8fd1-299b98288e55.8acd0166-0e33-4075-92e1-eaadc0687152 (peer_uuids:[]) 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 152 .mirror.primary.5afacd82-27c0-4e0f-8fd1-299b98288e55.66259543-e6a7-496c-980d-01258f005f8c (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:46:19.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:46:19.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:46:19.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:19.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:46:19.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:19.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:19.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:19.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:19.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:19.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:46:19.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:19.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:14' 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-03-23 19:46:14' 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:14' 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:19.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:19.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK1145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:46:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295154,"replay_state":"idle"}2026-03-23 19:45:55test_force_promote5afacd82-27c0-4e0f-8fd1-299b98288e55up+stoppedlocal image is primary45194524mirror.1vm08.local2026-03-23 19:46:19cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:46:15' 2026-03-23T19:46:19.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:19.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-03-23T19:46:19.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:19' 2026-03-23T19:46:19.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-03-23T19:46:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:46:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:46:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:46:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster1 mirror test_force_promote 100 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:46:19.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:46:20.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:46:20.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:46:20.068 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2083.33 bytes/sec: 8.1 MiB/s 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:46:20.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:46:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:46:20.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:46:20.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2272.72 bytes/sec: 8.9 MiB/s 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_force_promote 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:20.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:20.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_force_promote 2026-03-23T19:46:20.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:20.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:20.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:20.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote 2026-03-23T19:46:20.219 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote 2026-03-23T19:46:20.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:20.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_force_promote 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:20.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_force_promote 2026-03-23T19:46:20.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:20.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:20.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote 2026-03-23T19:46:20.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote 2026-03-23T19:46:20.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote 2026-03-23T19:46:21.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:21.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:21.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: cloned images' 2026-03-23T19:46:21.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:21.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:21.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:21 TEST: cloned images 2026-03-23T19:46:21.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:21 TEST: cloned images 2026-03-23T19:46:21.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - default' 2026-03-23T19:46:21.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:21.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:21 - default 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:21 - default 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ parent_image=test_parent 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ parent_snap=snap 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent test_parent snapshot 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:21.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror_parent test_parent 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:21.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror_parent/test_parent 2026-03-23T19:46:21.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror_parent test_parent snapshot 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:21.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent snapshot 2026-03-23T19:46:22.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:22.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-03-23T19:46:22.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_parent': 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15343d445e74 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15343d445e74 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:21 2026 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:21 2026 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:21 2026 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 4944e5d1-fb6a-4269-ba18-9e7163835ed4 2026-03-23T19:46:22.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror_parent test_parent 100 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:46:22.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror_parent/test_parent --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:46:22.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:46:22.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:46:22.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2000 bytes/sec: 7.8 MiB/s 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:22.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap 2026-03-23T19:46:23.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ protect_snapshot cluster2 mirror_parent test_parent snap 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:23.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap protect mirror_parent/test_parent@snap 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image=test_clone 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:23.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone --image-feature layering,exclusive-lock 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_clone 100 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_clone --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:46:23.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:46:24.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:46:24.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1587.3 bytes/sec: 6.2 MiB/s 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_clone snapshot 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:24.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone snapshot 2026-03-23T19:46:24.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:24.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_clone 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_clone': 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15467dc27db9 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15467dc27db9 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:23 2026 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:23 2026 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:23 2026 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: parent: mirror_parent/test_parent@snap 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: overlap: 128 MiB 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 10ee3e33-c76c-4e6e-bc79-1f655dfeedf6 2026-03-23T19:46:24.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror_parent test_parent 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror_parent test_parent started 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror_parent test_parent started 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/test_parent 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:25.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:25.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:25.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:25.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:25.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:25.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:25.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:25.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-03-23T19:46:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:46:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:46:25.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:25.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:46:25.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:46:25.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:46:25.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:25.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-03-23T19:46:26.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 5 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror_parent 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror_parent 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror_parent-test_parent.status 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror_parent-test_parent.status 2026-03-23T19:46:26.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:26.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.status 2026-03-23T19:46:26.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:26.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:26.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent test_parent /tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.status 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.status 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/test_parent --format xml 2026-03-23T19:46:26.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:26.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:46:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=5 2026-03-23T19:46:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:46:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:26.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:46:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:27.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-03-23T19:46:27.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:27.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:27.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=3 2026-03-23T19:46:27.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 5 = 3 2026-03-23T19:46:27.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:27.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:27.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-03-23T19:46:27.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:27.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=3 2026-03-23T19:46:27.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 5 = 3 2026-03-23T19:46:27.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:27.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:28.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-03-23T19:46:28.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:46:28.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:28.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:46:28.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:46:28.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:46:28.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:46:28.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 5 = '' 2026-03-23T19:46:28.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:28.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1.6 2026-03-23T19:46:29.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.status 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-03-23T19:46:29.991 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:30.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=5 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 5 = 5 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent test_parent 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:30.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:30.033 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying '' '' 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:31.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/test_parent 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_parent: 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4944e5d1-fb6a-4269-ba18-9e7163835ed4 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295182,"remote_snapshot_timestamp":1774295182,"replay_state":"idle"} 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:25 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:29' 2026-03-23T19:46:31.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:46:31.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:46:31.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:31.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295182,"remote_snapshot_timestamp":1774295182,"replay_state":"idle"} 2026-03-23T19:46:31.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:31.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:31.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:31.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:31.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:31.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:46:31.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:31.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:25' 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent test_parent up+replaying '2026-03-23 19:46:25' 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:25' 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:31.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-03-23T19:46:31.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK144024419mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094429mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184434mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234437mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_parent4944e5d1-fb6a-4269-ba18-9e7163835ed4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295182,"remote_snapshot_timestamp":1774295182,"replay_state":"idle"}44024419mirror.0vm08.local2026-03-23 19:46:25cluster2-DNEd9cbf5ec-1b4b-4db3-9373-4a97e373f41dup+stoppedlocal image is primary2026-03-23 19:46:29' 2026-03-23T19:46:31.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:31.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/last_update' 2026-03-23T19:46:31.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:25' 2026-03-23T19:46:31.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/state' 2026-03-23T19:46:31.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:46:31.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:31.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror_parent 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror_parent 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:31.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror_parent-test_parent.export 2026-03-23T19:46:31.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror_parent-test_parent.export 2026-03-23T19:46:31.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:31.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.export 2026-03-23T19:46:31.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:31.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror_parent/test_parent /tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.export 2026-03-23T19:46:31.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:46:31.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror_parent/test_parent /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:46:31.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_parent-test_parent.export 2026-03-23T19:46:31.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone started 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:31.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_clone started 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:32.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:32.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:32.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:32.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:32.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:32.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:32.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:32.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:32.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:32.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone 2026-03-23T19:46:32.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone'] 2026-03-23T19:46:32.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:32.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:32.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:32.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:32.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:32.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:32.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:32.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone 2026-03-23T19:46:32.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:32.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:46:32.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:46:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:46:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:46:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_clone 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_clone 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_clone 2026-03-23T19:46:33.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 154 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_clone 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:33.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_clone.status 2026-03-23T19:46:33.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_clone.status 2026-03-23T19:46:33.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:33.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.status 2026-03-23T19:46:33.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_clone.status 2026-03-23T19:46:33.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_clone.status 2026-03-23T19:46:33.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.status 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_clone /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.status 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.status 2026-03-23T19:46:33.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_clone --format xml 2026-03-23T19:46:33.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:33.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:46:33.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=154 2026-03-23T19:46:33.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:46:33.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:33.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.status 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.status 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-03-23T19:46:33.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:33.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:33.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=153 2026-03-23T19:46:33.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 154 = 153 2026-03-23T19:46:33.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:46:33.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:46:34.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.status 2026-03-23T19:46:34.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:34.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:34.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:34.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.status 2026-03-23T19:46:34.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:46:34.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-03-23T19:46:34.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=154 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 154 = 154 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_clone 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_clone up+replaying 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:34.167 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:35.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_clone up+replaying '' '' 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:35.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_clone 2026-03-23T19:46:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_clone: 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 10ee3e33-c76c-4e6e-bc79-1f655dfeedf6 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295184,"replay_state":"idle"} 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:30 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:46:34' 2026-03-23T19:46:35.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:46:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:46:35.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:46:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295184,"replay_state":"idle"} 2026-03-23T19:46:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:46:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:46:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:46:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:46:35.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:46:35.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:46:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:30' 2026-03-23T19:46:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_clone up+replaying '2026-03-23 19:46:30' 2026-03-23T19:46:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:35.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:46:30' 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:46:35.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:46:35.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:46:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:34test_clone10ee3e33-c76c-4e6e-bc79-1f655dfeedf6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295184,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:46:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:46:34' 2026-03-23T19:46:35.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:46:35.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/last_update' 2026-03-23T19:46:35.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:46:30' 2026-03-23T19:46:35.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/state' 2026-03-23T19:46:35.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:46:35.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:46:35.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_clone 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:46:35.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_clone.export 2026-03-23T19:46:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_clone.export 2026-03-23T19:46:35.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.export 2026-03-23T19:46:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_clone.export 2026-03-23T19:46:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_clone.export 2026-03-23T19:46:35.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:46:35.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.export 2026-03-23T19:46:35.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.export 2026-03-23T19:46:35.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_clone /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.export 2026-03-23T19:46:35.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:46:35.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_clone /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.export 2026-03-23T19:46:35.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:46:35.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.export 2026-03-23T19:46:35.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_clone.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_clone.export 2026-03-23T19:46:35.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_clone 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:35.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_clone 2026-03-23T19:46:35.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:35.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:35.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone 2026-03-23T19:46:35.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone 2026-03-23T19:46:35.731 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone 2026-03-23T19:46:36.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:36.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:36.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - clone v1' 2026-03-23T19:46:36.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:36.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:36 - clone v1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:36 - clone v1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap mirror test_clone1 snapshot 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image cluster1 mirror_parent test_parent snap mirror test_clone1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone1 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:36.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap mirror/test_clone1 --image-feature layering,exclusive-lock 2026-03-23T19:46:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:46:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:46:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:37.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster1 mirror test_clone1 snapshot 2026-03-23T19:46:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone1 2026-03-23T19:46:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:37.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone1 snapshot 2026-03-23T19:46:38.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:38.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 info mirror/test_clone1 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_clone1': 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15e43b31b4b2 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15e43b31b4b2 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:37 2026 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:37 2026 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:37 2026 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: parent: mirror_parent/test_parent@snap 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: overlap: 128 MiB 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: b457ef1c-8299-4402-a6cd-e8604c797093 2026-03-23T19:46:38.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap mirror test_clone_v1 snapshot --rbd-default-clone-format 1 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v1 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:46:38.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone_v1 --rbd-default-clone-format 1 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v1 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:38.169 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone_v1 --image-feature layering,exclusive-lock --rbd-default-clone-format 1 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_clone_v1 snapshot 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:38.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v1 snapshot 2026-03-23T19:46:38.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:38.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v1 2026-03-23T19:46:38.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_clone_v1': 2026-03-23T19:46:38.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1569cd8c8944 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1569cd8c8944 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:38 2026 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:38 2026 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:38 2026 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: parent: mirror_parent/test_parent@snap 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: overlap: 128 MiB 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 0d74b200-c269-4a38-9ad8-4fbaec91ca6d 2026-03-23T19:46:38.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_clone_format cluster2 mirror test_clone_v1 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v1 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v1 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk 'BEGIN { 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 1 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: $1 == "parent:" { 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: parent = $2 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: /op_features: .*clone-child/ { 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 2 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: END { 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: if (!parent) exit 1 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: print format 2026-03-23T19:46:38.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: }' 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v1 started 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:38.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:46:38.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:38.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:38.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_clone_v1 started 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:39.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:39.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:39.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:39.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:39.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:39.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:39.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:39.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:39.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:39.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:39.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:39.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-03-23T19:46:40.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:40.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:40.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:40.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:40.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:40.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:40.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:40.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-03-23T19:46:40.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:40.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:40.113 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:46:40.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:46:40.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:46:40.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:40.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:40.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:40.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-03-23T19:46:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:40.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:46:40.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:46:40.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:46:40.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:40.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:40.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:40.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-03-23T19:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:46:41.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:41.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:41.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-03-23T19:46:41.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:41.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:41.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:41.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:41.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:41.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:41.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:41.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-03-23T19:46:41.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:41.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:46:41.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:46:41.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:41.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_clone_format cluster1 mirror test_clone_v1 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v1 2026-03-23T19:46:41.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v1 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk 'BEGIN { 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 1 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: $1 == "parent:" { 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: parent = $2 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: /op_features: .*clone-child/ { 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 2 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: END { 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: if (!parent) exit 1 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: print format 2026-03-23T19:46:41.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: }' 2026-03-23T19:46:41.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 1 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_clone_v1 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:41.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_clone_v1 2026-03-23T19:46:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v1 2026-03-23T19:46:41.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v1 2026-03-23T19:46:41.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v1 2026-03-23T19:46:41.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_clone1 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone1 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:41.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:41.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_clone1 2026-03-23T19:46:41.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:41.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:41.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone1 2026-03-23T19:46:41.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone1 2026-03-23T19:46:41.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone1 2026-03-23T19:46:42.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot_retry cluster2 mirror_parent test_parent snap 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:42.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:42.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-03-23T19:46:42.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: unprotecting snap failed: (16) Device or resource busy 2026-03-23T19:46:42.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:42.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:43.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap 2026-03-23T19:46:43.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap 2026-03-23T19:46:43.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:46:43.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - clone v2' 2026-03-23T19:46:43.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:43.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:43.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:43 - clone v2 2026-03-23T19:46:43.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:43 - clone v2 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ parent_snap=snap_v2 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_v2 2026-03-23T19:46:43.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-03-23T19:46:44.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:44.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-03-23T19:46:46.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 7 2026-03-23T19:46:46.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap_v2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap_v2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v2 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:46:46.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock --rbd-default-clone-format 2 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_clone_v2 snapshot 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:46:46.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v2 snapshot 2026-03-23T19:46:47.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:46:47.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v2 2026-03-23T19:46:47.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_clone_v2': 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15887e62f57f 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15887e62f57f 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, operations 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: clone-child 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:46 2026 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:46 2026 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:46 2026 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: parent: mirror_parent/test_parent@snap_v2 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: overlap: 128 MiB 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 76ac3d37-9e1d-4fe6-ae46-e5ec0b6acb5c 2026-03-23T19:46:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:46:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_clone_format cluster2 mirror test_clone_v2 2026-03-23T19:46:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:46:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v2 2026-03-23T19:46:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v2 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk 'BEGIN { 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 1 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: $1 == "parent:" { 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: parent = $2 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: /op_features: .*clone-child/ { 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 2 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: END { 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: if (!parent) exit 1 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: print format 2026-03-23T19:46:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: }' 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 2 = 2 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v2 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v2 started 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:48.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_clone_v2 started 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:49.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:46:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:46:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:46:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:49.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:49.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:49.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:49.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:49.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-03-23T19:46:49.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:49.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:49.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:49.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:49.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-03-23T19:46:49.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:49.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:46:49.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:46:49.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:46:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:46:49.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-03-23T19:46:49.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:49.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:49.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:46:49.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:46:49.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:49.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:49.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:46:49.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:49.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-03-23T19:46:49.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:49.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:49.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:46:49.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:46:50.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:46:50.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:46:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:46:50.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:46:50.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:50.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:50.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:46:50.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:50.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-03-23T19:46:50.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:46:50.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:46:50.468 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:46:50.469 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:46:50.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:50.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:50.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:46:50.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-03-23T19:46:50.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:46:50.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:46:50.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:46:50.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:46:50.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:46:50.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:46:50.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:46:50.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_clone_format cluster1 mirror test_clone_v2 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v2 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v2 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk 'BEGIN { 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 1 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: $1 == "parent:" { 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: parent = $2 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: /op_features: .*clone-child/ { 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: format = 2 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: } 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: END { 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: if (!parent) exit 1 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: print format 2026-03-23T19:46:50.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: }' 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 2 = 2 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_v2 2026-03-23T19:46:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-03-23T19:46:50.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:50.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-03-23T19:46:50.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 8 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-03-23T19:46:50.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:46:50.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 6 396b2a60-b887-445c-bab9-5da3c757211a 128 MiB Mon Mar 23 19:46:44 2026 trash (user snap_v2) 2026-03-23T19:46:50.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:50.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:50.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:46:50.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:51.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-03-23T19:46:51.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:46:51.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ continue 2026-03-23T19:46:51.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:46:51.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-03-23T19:46:53.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 6 46c9928c-fc80-4d37-b665-99b39e55ef6f 128 MiB Mon Mar 23 19:46:46 2026 trash (user snap_v2) 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_clone_v2 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:46:53.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:46:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_clone_v2 2026-03-23T19:46:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:53.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v2 2026-03-23T19:46:53.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v2 2026-03-23T19:46:55.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test_clone_v2 deleted 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v2 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-03-23T19:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:46:55.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-03-23T19:46:55.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:46:55.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:46:55.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test_clone_v2 deleted '' 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:46:56.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_clone_v2 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-03-23T19:46:56.071 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:46:56.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-03-23T19:46:56.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-03-23T19:46:56.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:46:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:46:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:46:56.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-03-23T19:46:57.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:46:57.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:57.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:46:57.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - clone v2 non-primary' 2026-03-23T19:46:57.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:46:57.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:46:57 - clone v2 non-primary 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:46:57 - clone v2 non-primary 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_v2 2026-03-23T19:46:57.177 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-03-23T19:46:57.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:46:57.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-03-23T19:47:00.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 10 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:00.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:01.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_parent': 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 2 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1143274df298 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1143274df298 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:46:24 2026 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:46:24 2026 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:46:24 2026 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: False 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 4944e5d1-fb6a-4269-ba18-9e7163835ed4 2026-03-23T19:47:01.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:47:01.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap_v2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ clone_image cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_pool=mirror_parent 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_image=test_parent 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local parent_snap=snap_v2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_pool=mirror 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local clone_image=test_clone_v2 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 6 2026-03-23T19:47:01.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock --rbd-default-clone-format 2 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster1 mirror test_clone_v2 snapshot 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:47:01.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:01.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone_v2 snapshot 2026-03-23T19:47:02.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:02.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_clone_v2': 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 160b10a9dce2 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.160b10a9dce2 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:02.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, operations 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: clone-child 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:01 2026 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:01 2026 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:01 2026 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: parent: mirror_parent/test_parent@snap_v2 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: overlap: 128 MiB 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 82f6a236-3ca6-4dde-ba93-1db94c4403d1 2026-03-23T19:47:02.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_v2 2026-03-23T19:47:02.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-03-23T19:47:03.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:47:03.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:47:03.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-03-23T19:47:03.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:03.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:03.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:03.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:47:03.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-03-23T19:47:03.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:03.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-03-23T19:47:05.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 11 2026-03-23T19:47:05.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:05.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:05.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:05.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:06.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:47:06.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-03-23T19:47:06.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 9 754ac0c8-93cf-44f0-8d0d-614beb65b665 128 MiB Mon Mar 23 19:46:59 2026 trash (user snap_v2) 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_clone_v2 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:47:06.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:47:06.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_clone_v2 2026-03-23T19:47:06.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:06.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:06.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_clone_v2 2026-03-23T19:47:06.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone_v2 2026-03-23T19:47:06.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone_v2 2026-03-23T19:47:07.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:47:07.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:07.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:07.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:07.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:07.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_v2 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-03-23T19:47:08.594 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -F ' trash (user snap_v2)' 2026-03-23T19:47:08.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:08.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror_parent test_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror_parent test_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_parent 2026-03-23T19:47:08.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror_parent/test_parent 2026-03-23T19:47:08.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror_parent/test_parent 2026-03-23T19:47:09.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:47:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:09.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: data pool' 2026-03-23T19:47:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:47:09.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:47:09 TEST: data pool 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:47:09 TEST: data pool 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ dp_image=test_data_pool 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_data_pool snapshot 128 --data-pool mirror_parent 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_data_pool 128 --data-pool mirror_parent 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 128 ']' 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=128 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:09.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock --data-pool mirror_parent mirror/test_data_pool 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_data_pool snapshot 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:09.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_data_pool snapshot 2026-03-23T19:47:09.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:09.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_data_pool 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_data_pool': 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15b8a2f2bcdf 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: data_pool: mirror_parent 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.3.15b8a2f2bcdf 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, data-pool 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:09 2026 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:09 2026 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:09 2026 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 67c26ee2-1903-4e57-be7f-4ab3e54dc8a3 2026-03-23T19:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_data_pool cluster2 mirror test_data_pool 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_data_pool 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 info mirror/test_data_pool 2026-03-23T19:47:09.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ data_pool=mirror_parent 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test mirror_parent = mirror_parent 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_data_pool 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_data_pool started 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:47:09.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_data_pool started 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:47:10.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:47:10.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:47:10.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:47:10.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:47:10.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:47:10.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:10.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:10.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:10.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-03-23T19:47:10.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:47:10.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:47:10.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:47:10.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:47:10.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:10.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:10.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:10.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-03-23T19:47:10.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:47:10.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:47:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:47:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:47:10.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:47:10.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:47:10.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:47:10.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-03-23T19:47:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:47:10.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:47:10.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:47:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:47:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:47:10.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:47:10.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:47:10.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:47:10.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:47:10.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:10.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-03-23T19:47:10.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:47:10.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:47:10.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:47:10.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:47:11.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:47:11.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:11.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:47:11.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:47:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:47:11.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:11.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-03-23T19:47:11.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:47:11.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:47:11.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:47:11.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:47:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:11.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:47:11.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-03-23T19:47:11.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:47:11.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:47:11.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:47:11.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:47:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:47:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:47:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:47:11.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_data_pool cluster1 mirror test_data_pool 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_data_pool 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 info mirror/test_data_pool 2026-03-23T19:47:11.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ data_pool=mirror_parent 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test mirror_parent = mirror_parent 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test_data_pool snap1 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:12.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap1 2026-03-23T19:47:12.112 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:12.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:12.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:12.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:12.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2325.58 bytes/sec: 9.1 MiB/s 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test_data_pool snap2 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:12.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap2 2026-03-23T19:47:13.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:13.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:13.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:13.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:13.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2173.91 bytes/sec: 8.5 MiB/s 2026-03-23T19:47:13.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_data_pool 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_data_pool 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:13.218 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_data_pool 2026-03-23T19:47:14.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 15 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_data_pool 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_data_pool.status 2026-03-23T19:47:14.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_data_pool.status 2026-03-23T19:47:14.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:14.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.status 2026-03-23T19:47:14.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_data_pool /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.status 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:14.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.status 2026-03-23T19:47:14.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_data_pool --format xml 2026-03-23T19:47:14.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:47:14.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:47:14.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=15 2026-03-23T19:47:14.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:47:14.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:47:14.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-03-23T19:47:14.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:47:14.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:47:14.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=12 2026-03-23T19:47:14.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 15 = 12 2026-03-23T19:47:14.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:47:14.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-03-23T19:47:14.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:47:14.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:47:14.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 15 = '' 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:47:14.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-03-23T19:47:15.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:47:15.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:47:15.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:47:15.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:47:15.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:47:15.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:47:15.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:47:15.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 15 = '' 2026-03-23T19:47:15.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:47:15.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1.6 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.status 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-03-23T19:47:17.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:47:17.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=15 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 15 = 15 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_data_pool 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_data_pool up+replaying 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:47:17.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:18.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_data_pool up+replaying '' '' 2026-03-23T19:47:18.348 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:47:18.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_data_pool 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_data_pool: 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 67c26ee2-1903-4e57-be7f-4ab3e54dc8a3 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295229,"replay_state":"idle"} 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:47:11 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:47:14' 2026-03-23T19:47:18.401 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:47:18.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:47:18.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:47:18.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295229,"replay_state":"idle"} 2026-03-23T19:47:18.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:47:18.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:47:18.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:47:18.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:47:18.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:47:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:47:18.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:47:18.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:47:18.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:47:11' 2026-03-23T19:47:18.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_data_pool up+replaying '2026-03-23 19:47:11' 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:47:11' 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:47:18.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:47:18.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK244024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:46:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:47:14test_data_pool67c26ee2-1903-4e57-be7f-4ab3e54dc8a3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295229,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:47:11cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:47:14' 2026-03-23T19:47:18.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:47:18.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-03-23T19:47:18.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:47:11' 2026-03-23T19:47:18.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-03-23T19:47:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:47:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:47:18.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:47:18.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool@snap1 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool@snap1 2026-03-23T19:47:18.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.460 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:18.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:18.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap1 /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:18.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap1 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:18.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap1.export 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool@snap2 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool@snap2 2026-03-23T19:47:18.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:18.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:18.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap2 /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap2.export 2026-03-23T19:47:18.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:18.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap2 /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:19.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:19.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:19.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool@snap2.export 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_data_pool.export 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_data_pool.export 2026-03-23T19:47:19.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:19.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.export 2026-03-23T19:47:19.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:47:19.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.export 2026-03-23T19:47:19.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:19.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:47:19.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_data_pool.export 2026-03-23T19:47:19.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_data_pool 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_data_pool 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:47:19.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_data_pool 2026-03-23T19:47:21.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-23T19:47:21.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_data_pool 2026-03-23T19:47:22.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:47:22.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:22.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: disable mirroring / delete non-primary image' 2026-03-23T19:47:22.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:47:22.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:47:22.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:47:22 TEST: disable mirroring / delete non-primary image 2026-03-23T19:47:22.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:47:22 TEST: disable mirroring / delete non-primary image 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image2=test2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image3=test3 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image4=test4 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image5=test5 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test2 snapshot 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:47:22.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test2 2026-03-23T19:47:23.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test2 snapshot 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:23.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-03-23T19:47:23.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:23.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-03-23T19:47:23.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test2': 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15e4f8f4a45c 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15e4f8f4a45c 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 1403fc62-e876-4eff-80f2-75c37478f355 2026-03-23T19:47:23.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test2 100 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:23.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:23.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:24.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2702.7 bytes/sec: 11 MiB/s 2026-03-23T19:47:24.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test2 snap1 2026-03-23T19:47:24.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:24.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:24.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:24.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:24.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap1 2026-03-23T19:47:24.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test2 snap2 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:24.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap2 2026-03-23T19:47:25.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test2 = test4 ']' 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test2 = test5 ']' 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test2 100 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:25.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:25.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:26.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:26.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-03-23T19:47:27.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 161 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:27.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test2 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test2 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-03-23T19:47:27.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=113e70460e3d 2026-03-23T19:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:27.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test2 present 113e70460e3d 2026-03-23T19:47:27.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:27.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=113e70460e3d 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test2 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test2 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-03-23T19:47:27.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=113e70460e3d 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 113e70460e3d 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 113e70460e3d -o 113e70460e3d = 113e70460e3d 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:27.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:27.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:27.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap2 2026-03-23T19:47:27.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:27.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:27.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:28.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-03-23T19:47:28.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:error setting snapshot context: (2) No such file or directory 2026-03-23T19:47:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ continue 2026-03-23T19:47:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:28.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:47:30.284 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:47:30.282+0000 7fedd70f0640 -1 reset not still connected to 0x563414f152b0 2026-03-23T19:47:30.290 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:47:30.282+0000 7fedd70f0640 -1 reset not still connected to 0x56341687b1e0 2026-03-23T19:47:30.291 INFO:tasks.ceph.cluster2.osd.2.vm08.stderr:2026-03-23T19:47:30.282+0000 7fedd70f0640 -1 reset not still connected to 0x563416c17380 2026-03-23T19:47:30.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-03-23T19:47:30.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test2': 2026-03-23T19:47:30.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 4 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 113e70460e3d 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.113e70460e3d 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:25 2026 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:25 2026 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:25 2026 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: False 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 1403fc62-e876-4eff-80f2-75c37478f355 2026-03-23T19:47:30.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:47:30.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test3 snapshot 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test3 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:47:30.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test3 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test3 snapshot 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:30.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:30.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:30.483 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test3 snapshot 2026-03-23T19:47:31.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:31.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test3 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test3': 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15f9cf3f7e40 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15f9cf3f7e40 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:30 2026 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:30 2026 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:30 2026 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 6e50175d-6464-4bfb-b0fd-ddc517319eca 2026-03-23T19:47:31.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test3 100 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:31.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:31.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:31.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:31.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2083.33 bytes/sec: 8.1 MiB/s 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test3 snap1 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:31.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap1 2026-03-23T19:47:32.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test3 snap2 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:32.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap2 2026-03-23T19:47:33.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test3 = test4 ']' 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test3 = test5 ']' 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test3 100 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:33.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:33.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test3 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:33.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test3 2026-03-23T19:47:34.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 165 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test3 present 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test3 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test3 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-03-23T19:47:34.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=114c6f5e61b2 2026-03-23T19:47:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:34.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test3 present 114c6f5e61b2 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=114c6f5e61b2 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test3 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test3 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-03-23T19:47:34.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=114c6f5e61b2 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 114c6f5e61b2 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 114c6f5e61b2 -o 114c6f5e61b2 = 114c6f5e61b2 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test3 snap2 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap2 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:34.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:35.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test3': 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 3 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 114c6f5e61b2 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.114c6f5e61b2 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:33 2026 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:33 2026 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:33 2026 2026-03-23T19:47:35.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: False 2026-03-23T19:47:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 6e50175d-6464-4bfb-b0fd-ddc517319eca 2026-03-23T19:47:35.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test4 snapshot 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test4 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:47:35.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test4 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test4 snapshot 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:35.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test4 snapshot 2026-03-23T19:47:36.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:36.240 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test4 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test4': 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 160f9ca6f2c8 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.160f9ca6f2c8 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:35 2026 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:35 2026 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:35 2026 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: ce7a23bb-fa1e-40b7-a31e-1df539afb57b 2026-03-23T19:47:36.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test4 100 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:36.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:36.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:36.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:36.369 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2083.33 bytes/sec: 8.1 MiB/s 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test4 snap1 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:36.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap1 2026-03-23T19:47:37.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:37.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test4 snap2 2026-03-23T19:47:37.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:37.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:37.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:37.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:37.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap2 2026-03-23T19:47:38.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test4 = test4 ']' 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ protect_snapshot cluster2 mirror test4 snap1 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:38.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap1 2026-03-23T19:47:38.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ protect_snapshot cluster2 mirror test4 snap2 2026-03-23T19:47:38.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:38.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:38.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:38.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:38.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap2 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test4 100 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:38.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:38.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:38.368 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1960.78 bytes/sec: 7.7 MiB/s 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test4 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:38.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test4 2026-03-23T19:47:39.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 169 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:39.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test4 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test4 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-03-23T19:47:39.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=1150af5c706b 2026-03-23T19:47:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:39.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test4 present 1150af5c706b 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1150af5c706b 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:39.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test4 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test4 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-03-23T19:47:39.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1150af5c706b 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1150af5c706b 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1150af5c706b -o 1150af5c706b = 1150af5c706b 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap2 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:39.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:40.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test4': 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 3 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1150af5c706b 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1150af5c706b 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:38 2026 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:38 2026 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:38 2026 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: True 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: ce7a23bb-fa1e-40b7-a31e-1df539afb57b 2026-03-23T19:47:40.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:47:40.466 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test5 snapshot 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test5 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:47:40.467 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test5 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test5 snapshot 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:40.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test5 snapshot 2026-03-23T19:47:41.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:41.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test5 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test5': 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 162ce78c82ab 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.162ce78c82ab 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:40 2026 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:40 2026 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:40 2026 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: c778b56a-d24c-430f-9bbc-120a4bc54a50 2026-03-23T19:47:41.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test5 100 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:41.291 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:41.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:41.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:41.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test5 snap1 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:41.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap1 2026-03-23T19:47:42.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test5 snap2 2026-03-23T19:47:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:42.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:42.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap2 2026-03-23T19:47:43.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test5 = test4 ']' 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' test5 = test5 ']' 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ protect_snapshot cluster2 mirror test5 snap1 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:43.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:43.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap1 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ protect_snapshot cluster2 mirror test5 snap2 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:43.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap2 2026-03-23T19:47:43.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test5 100 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:47:43.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:47:43.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1785.71 bytes/sec: 7.0 MiB/s 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test5 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:43.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test5 2026-03-23T19:47:44.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 173 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test5 present 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test5 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test5 2026-03-23T19:47:44.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-03-23T19:47:44.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=1137215e5856 2026-03-23T19:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test5 present 1137215e5856 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1137215e5856 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test5 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:44.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test5 2026-03-23T19:47:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-03-23T19:47:44.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1137215e5856 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1137215e5856 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1137215e5856 -o 1137215e5856 = 1137215e5856 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test5 snap2 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap2 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:44.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:45.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test5': 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 3 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1137215e5856 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1137215e5856 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:43 2026 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:43 2026 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:43 2026 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: True 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:45.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:45.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: c778b56a-d24c-430f-9bbc-120a4bc54a50 2026-03-23T19:47:45.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:47:45.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:45.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-03-23T19:47:45.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:45.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:45.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=image 2026-03-23T19:47:45.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-03-23T19:47:45.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd: mirroring is already configured for image mode 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image4} 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ disable_mirror cluster2 mirror test2 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:45.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test2 2026-03-23T19:47:46.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring disabled 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image4} 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ disable_mirror cluster2 mirror test4 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:46.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test4 2026-03-23T19:47:47.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring disabled 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror test5 snap1 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:47.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap1 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror test5 snap2 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:47.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap2 2026-03-23T19:47:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image3} ${image5} 2026-03-23T19:47:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror test3 snap1 2026-03-23T19:47:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:47.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:47.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:47.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:47.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap1 2026-03-23T19:47:48.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror test3 snap2 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:48.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap2 2026-03-23T19:47:49.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test3 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:47:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:47:49.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test3 2026-03-23T19:47:49.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:49.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:49.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:49.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test3 2026-03-23T19:47:49.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test3 2026-03-23T19:47:50.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:47:50.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:50.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image3} ${image5} 2026-03-23T19:47:50.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror test5 snap1 2026-03-23T19:47:50.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:50.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:50.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:50.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:47:50.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap1 2026-03-23T19:47:51.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_snapshot cluster2 mirror test5 snap2 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:47:51.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap2 2026-03-23T19:47:52.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing snap: 100% complete...done. 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test5 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:47:52.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:47:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test5 2026-03-23T19:47:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:52.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test5 2026-03-23T19:47:52.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test5 2026-03-23T19:47:53.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test2 deleted 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:53.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test2 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test2 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-03-23T19:47:53.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:53.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test2: (2) No such file or directory 2026-03-23T19:47:53.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:47:53.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:53.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test2 deleted '' 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:53.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test2 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test2 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-03-23T19:47:53.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:53.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test2: (2) No such file or directory 2026-03-23T19:47:53.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:47:53.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test3 deleted 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test3 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test3 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-03-23T19:47:53.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:53.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test3: (2) No such file or directory 2026-03-23T19:47:53.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:47:53.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:53.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test3 deleted '' 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test3 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test3 2026-03-23T19:47:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test3 2026-03-23T19:47:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-03-23T19:47:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:53.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test3: (2) No such file or directory 2026-03-23T19:47:53.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test4 deleted 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test4 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test4 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-03-23T19:47:53.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:54.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test4: (2) No such file or directory 2026-03-23T19:47:54.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:47:54.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:54.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:54.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test4 deleted '' 2026-03-23T19:47:54.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:54.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:54.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:47:54.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test4 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test4 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-03-23T19:47:54.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:54.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test4: (2) No such file or directory 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test5 deleted 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:54.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test5 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test5 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-03-23T19:47:54.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:54.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test5: (2) No such file or directory 2026-03-23T19:47:54.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:47:54.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:47:54.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test5 deleted '' 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test5 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:47:54.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test5 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test5 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-03-23T19:47:54.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:47:54.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test5: (2) No such file or directory 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:47:54.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: snapshot rename' 2026-03-23T19:47:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:47:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:47:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:47:54 TEST: snapshot rename 2026-03-23T19:47:54.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:47:54 TEST: snapshot rename 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ snap_name=snap_rename 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test2 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:47:54.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-03-23T19:47:54.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:47:54.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-03-23T19:47:54.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test2': 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 3 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 15e4f8f4a45c 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.15e4f8f4a45c 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:23 2026 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 6d467410-1fcf-4370-afe5-a05a1e79e901 2026-03-23T19:47:54.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test2 snap_rename_0 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_0 2026-03-23T19:47:54.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap_rename_0 2026-03-23T19:47:55.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:47:55.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 1 20 2026-03-23T19:47:55.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 1 - 1 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_0 snap_rename_1 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_0 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_1 2026-03-23T19:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_0 mirror/test2@snap_rename_1 2026-03-23T19:47:55.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 2 - 1 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_1 snap_rename_2 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_1 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_2 2026-03-23T19:47:55.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_1 mirror/test2@snap_rename_2 2026-03-23T19:47:55.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 3 - 1 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_2 snap_rename_3 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_2 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_3 2026-03-23T19:47:55.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_2 mirror/test2@snap_rename_3 2026-03-23T19:47:55.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 4 - 1 2026-03-23T19:47:55.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_3 snap_rename_4 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_3 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_4 2026-03-23T19:47:55.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_3 mirror/test2@snap_rename_4 2026-03-23T19:47:55.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 5 - 1 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_4 snap_rename_5 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_4 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_5 2026-03-23T19:47:55.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_4 mirror/test2@snap_rename_5 2026-03-23T19:47:55.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 6 - 1 2026-03-23T19:47:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_5 snap_rename_6 2026-03-23T19:47:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_5 2026-03-23T19:47:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_6 2026-03-23T19:47:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_5 mirror/test2@snap_rename_6 2026-03-23T19:47:55.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:55.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 7 - 1 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_6 snap_rename_7 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_6 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_7 2026-03-23T19:47:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_6 mirror/test2@snap_rename_7 2026-03-23T19:47:56.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 8 - 1 2026-03-23T19:47:56.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_7 snap_rename_8 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_7 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_8 2026-03-23T19:47:56.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_7 mirror/test2@snap_rename_8 2026-03-23T19:47:56.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 9 - 1 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_8 snap_rename_9 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_8 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_9 2026-03-23T19:47:56.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_8 mirror/test2@snap_rename_9 2026-03-23T19:47:56.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 10 - 1 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_9 snap_rename_10 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_9 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_10 2026-03-23T19:47:56.163 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_9 mirror/test2@snap_rename_10 2026-03-23T19:47:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 11 - 1 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_10 snap_rename_11 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_10 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_11 2026-03-23T19:47:56.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_10 mirror/test2@snap_rename_11 2026-03-23T19:47:56.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 12 - 1 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_11 snap_rename_12 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_11 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_12 2026-03-23T19:47:56.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_11 mirror/test2@snap_rename_12 2026-03-23T19:47:56.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 13 - 1 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_12 snap_rename_13 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_12 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_13 2026-03-23T19:47:56.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_12 mirror/test2@snap_rename_13 2026-03-23T19:47:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 14 - 1 2026-03-23T19:47:56.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_13 snap_rename_14 2026-03-23T19:47:56.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_13 2026-03-23T19:47:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_14 2026-03-23T19:47:56.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_13 mirror/test2@snap_rename_14 2026-03-23T19:47:56.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 15 - 1 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_14 snap_rename_15 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_14 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_15 2026-03-23T19:47:56.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_14 mirror/test2@snap_rename_15 2026-03-23T19:47:56.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 16 - 1 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_15 snap_rename_16 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_15 2026-03-23T19:47:56.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_16 2026-03-23T19:47:56.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_15 mirror/test2@snap_rename_16 2026-03-23T19:47:56.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 17 - 1 2026-03-23T19:47:56.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_16 snap_rename_17 2026-03-23T19:47:56.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_16 2026-03-23T19:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_17 2026-03-23T19:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_16 mirror/test2@snap_rename_17 2026-03-23T19:47:56.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 18 - 1 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_17 snap_rename_18 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_17 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_18 2026-03-23T19:47:56.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_17 mirror/test2@snap_rename_18 2026-03-23T19:47:56.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 19 - 1 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_18 snap_rename_19 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_18 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_19 2026-03-23T19:47:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_18 mirror/test2@snap_rename_19 2026-03-23T19:47:56.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in `seq 1 20` 2026-03-23T19:47:56.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ expr 20 - 1 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_19 snap_rename_20 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap_rename_19 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local new_snap=snap_rename_20 2026-03-23T19:47:56.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_19 mirror/test2@snap_rename_20 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-03-23T19:47:57.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 176 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test2 snap_rename_20 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap_rename_20 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:57.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:47:58.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-03-23T19:47:58.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:error setting snapshot context: (2) No such file or directory 2026-03-23T19:47:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ continue 2026-03-23T19:47:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:47:58.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:48:00.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test2': 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 5 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 113ec8f121ee 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.113ec8f121ee 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, operations, non-primary 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: snap-trash 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:47:55 2026 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:47:55 2026 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:47:55 2026 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: False 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 6d467410-1fcf-4370-afe5-a05a1e79e901 2026-03-23T19:48:00.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror test4 snap1 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:48:00.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap1 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ unprotect_snapshot cluster2 mirror test4 snap2 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap2 2026-03-23T19:48:00.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap2 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image4} 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test2 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:01.017 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test2 2026-03-23T19:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test2 2026-03-23T19:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test2 2026-03-23T19:48:03.938 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing all snapshots: 33% complete... Removing all snapshots: 66% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-23T19:48:03.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test2 2026-03-23T19:48:05.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for i in ${image2} ${image4} 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test4 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:05.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test4 2026-03-23T19:48:05.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:05.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:05.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test4 2026-03-23T19:48:05.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test4 2026-03-23T19:48:06.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-23T19:48:06.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test4 2026-03-23T19:48:07.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:48:07.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:07.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:48:07.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: demote image while daemon is offline' 2026-03-23T19:48:07.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:48:07.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:48:07.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:48:07 TEST: demote image while daemon is offline 2026-03-23T19:48:07.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:48:07 TEST: demote image while daemon is offline 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image=test_demote_image 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_demote_image 2026-03-23T19:48:07.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:07.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_demote_image 2026-03-23T19:48:07.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:48:07.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:07.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:48:07.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-03-23T19:48:07.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:48:07.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_demote_image': 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 16c574d815b7 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.16c574d815b7 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:48:07 2026 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:48:07 2026 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:48:07 2026 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:08.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:48:08.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:48:08.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:48:08.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:48:08.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:08.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:48:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:09.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-03-23T19:48:09.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:09.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:09.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:09.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:09.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:09.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:09.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:09.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:09.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:09.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:09.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:09.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:09.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:09.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:09.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:09.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:09.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:48:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:09.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:09.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:09.334 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:09.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:48:09.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:48:09.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:48:09.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:48:09.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:48:09.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:09.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:09.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:09.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:09.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:09.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:48:10.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:10.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:10.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:10.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:10.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:10.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:10.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:10.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:10.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:10.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:10.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:10.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:10.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:10.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:48:10.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:48:10.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:10.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:10.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:11.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:48:11.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:48:11.596 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:11.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:11.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:11.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:11.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:11.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:11.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:48:11.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:11.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:11.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:11.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:11.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:11.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:48:11.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:48:11.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:11.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:11.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-03-23T19:48:12.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 178 2026-03-23T19:48:12.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-03-23T19:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:12.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:12.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:12.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:12.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:48:12.143 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=178 2026-03-23T19:48:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:48:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:12.144 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:12.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:12.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=177 2026-03-23T19:48:12.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 178 = 177 2026-03-23T19:48:12.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:12.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:12.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:12.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=177 2026-03-23T19:48:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 178 = 177 2026-03-23T19:48:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:12.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:13.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:13.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=178 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 178 = 178 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:13.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:14.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:14.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295287,"replay_state":"idle"} 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:10 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:48:14.738 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:48:14.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:48:14.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:14.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295287,"replay_state":"idle"} 2026-03-23T19:48:14.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:14.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:14.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:14.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:10' 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-03-23 19:48:10' 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:10' 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:14.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1144024407mirror.0vm08.local20.2.0-712-g70f8415btrueOK44094414mirror.1vm08.local20.2.0-712-g70f8415bfalseOK44184428mirror.2vm08.local20.2.0-712-g70f8415bfalseOK44234432mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}44024407mirror.0vm08.local2026-03-23 19:47:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:09test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295287,"replay_state":"idle"}44094414mirror.1vm08.local2026-03-23 19:48:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:48:14.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:14.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:14.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:10' 2026-03-23T19:48:14.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:14.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:48:14.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:48:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:14.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:14.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:15.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:15.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:48:15.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:15.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:15.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:15.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:14 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295287,"replay_state":"idle"} 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:10 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 177 .mirror.primary.0d382ca5-c95d-4b0a-9264-c4cd9429ae17.71721f80-ffa5-4930-a6d4-70017a91188d (peer_uuids:[]) 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 178 .mirror.primary.0d382ca5-c95d-4b0a-9264-c4cd9429ae17.1039e875-e2e8-42fe-9069-8674d29f6ce7 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:48:15.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:15.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:48:15.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:15.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:15.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:15.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:15.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:15.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:15.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:15.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:15.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:14' 2026-03-23T19:48:15.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-03-23 19:48:14' 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:14' 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:15.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:47:55test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+stoppedlocal image is primary45194524mirror.1vm08.local2026-03-23 19:48:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295287,"replay_state":"idle"}2026-03-23 19:48:10' 2026-03-23T19:48:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:15.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:15.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:14' 2026-03-23T19:48:15.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:15.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:48:15.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:15.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:48:15.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:15.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:48:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:48:15.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:15.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:15.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:48:15.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:48:15.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:48:15.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=68023 2026-03-23T19:48:15.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 68023 ']' 2026-03-23T19:48:15.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 68023 2026-03-23T19:48:15.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:15.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:15.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:15.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68023 '$2 == pid {print; exit 1}' 2026-03-23T19:48:15.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 68023 0.3 0.7 825428 74996 ? Ssl 19:32 0:03 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:15.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:15.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:16.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68023 '$2 == pid {print; exit 1}' 2026-03-23T19:48:16.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:48:16.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:16.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68023 '$2 == pid {print; exit 1}' 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:16.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:16.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:48:16.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:16.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:16.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:16.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:16.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:48:17.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:17.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:17.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:17.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:48:17.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:48:17.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:17.024 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:17.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:48:17.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:48:17.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:48:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=68067 2026-03-23T19:48:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 68067 ']' 2026-03-23T19:48:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 68067 2026-03-23T19:48:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:17.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:17.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:17.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68067 '$2 == pid {print; exit 1}' 2026-03-23T19:48:17.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 68067 0.2 0.6 737324 66632 ? Ssl 19:32 0:01 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:17.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:17.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:18.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:18.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68067 '$2 == pid {print; exit 1}' 2026-03-23T19:48:18.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:48:18.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:18.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68067 '$2 == pid {print; exit 1}' 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:48:18.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:18.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:48:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:18.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:18.101 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:48:18.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:18.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:18.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:48:18.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:48:18.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:48:18.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:18.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:48:18.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:48:18.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:48:18.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:18.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:48:18.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:48:18.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:48:18.145 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:48:18.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=68119 2026-03-23T19:48:18.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 68119 ']' 2026-03-23T19:48:18.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 68119 2026-03-23T19:48:18.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:18.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:18.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:18.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68119 '$2 == pid {print; exit 1}' 2026-03-23T19:48:18.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 68119 0.1 0.6 737324 64620 ? Ssl 19:32 0:01 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:18.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:18.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:19.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:19.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68119 '$2 == pid {print; exit 1}' 2026-03-23T19:48:19.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:48:19.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:19.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68119 '$2 == pid {print; exit 1}' 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:19.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:48:19.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:19.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:19.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:19.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:19.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:19.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:19.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:19.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:48:19.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:19.216 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:19.217 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:48:19.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:48:19.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:19.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:48:19.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:48:19.247 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:48:19.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:48:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=68205 2026-03-23T19:48:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 68205 ']' 2026-03-23T19:48:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 68205 2026-03-23T19:48:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:19.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:19.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:19.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68205 '$2 == pid {print; exit 1}' 2026-03-23T19:48:19.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 68205 0.1 0.6 737328 64280 ? Ssl 19:32 0:01 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:19.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:19.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:20.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:20.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68205 '$2 == pid {print; exit 1}' 2026-03-23T19:48:20.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:48:20.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:48:20.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=68205 '$2 == pid {print; exit 1}' 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:20.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:48:20.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:20.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:20.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:20.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:48:20.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:20.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:20.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:48:20.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:48:20.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:48:20.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:48:20.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:48:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:48:20.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2000 bytes/sec: 7.8 MiB/s 2026-03-23T19:48:20.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test_demote_image 2026-03-23T19:48:20.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:20.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:20.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:20.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-03-23T19:48:22.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:48:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:48:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:22.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:22.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:22.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:48:22.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:22.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:22.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-03-23T19:48:22.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:22.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:22.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-03-23T19:48:22.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-03-23T19:48:22.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:22.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:22.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:48:22.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=179 2026-03-23T19:48:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:48:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:22.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:48:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:22.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:22.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:22.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:22.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:22.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:22.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:22.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=178 2026-03-23T19:48:22.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 179 = 178 2026-03-23T19:48:22.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:22.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:23.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:23.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:23.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=178 2026-03-23T19:48:23.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 179 = 178 2026-03-23T19:48:23.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:23.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:48:23.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:23.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:23.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:23.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:23.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:23.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=178 2026-03-23T19:48:23.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 179 = 178 2026-03-23T19:48:23.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:23.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1.6 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:25.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:25.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=178 2026-03-23T19:48:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 179 = 178 2026-03-23T19:48:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:25.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:48:27.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:27.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=179 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 179 = 179 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:27.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:28.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:28.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:28.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:24 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:28' 2026-03-23T19:48:28.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:48:28.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:48:28.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:28.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:48:28.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:28.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:28.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:28.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:28.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:28.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:28.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:28.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:24' 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-03-23 19:48:24' 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:24' 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:28.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:28.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:26test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+unknownremote image is not primary45194524mirror.1vm08.local2026-03-23 19:48:24cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:48:28' 2026-03-23T19:48:28.722 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:28.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:28.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:24' 2026-03-23T19:48:28.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:28.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:48:28.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:28.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:48:28.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:28.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:29.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:28 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:24' 2026-03-23T19:48:29.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:48:29.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:48:29.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:29.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image demoted 2026-03-23T19:48:29.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:29.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:29.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:29.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:29.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:29.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:28' 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-03-23 19:48:28' 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:28' 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:29.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:29.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1157785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:26cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:24test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+unknownremote image demoted57845791mirror.1vm08.local2026-03-23 19:48:28cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:48:24' 2026-03-23T19:48:29.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:29.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:29.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:28' 2026-03-23T19:48:29.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:29.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:48:29.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:48:29.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-03-23T19:48:29.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-03-23T19:48:29.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:29.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export 2026-03-23T19:48:29.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-03-23T19:48:29.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-03-23T19:48:29.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:29.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:48:29.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:48:29.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export 2026-03-23T19:48:30.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:48:30.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:48:30.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:48:30.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:48:30.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:48:30.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:30.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_demote_image 2026-03-23T19:48:30.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:30.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:30.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:30.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:48:30.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-03-23T19:48:31.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:31.117 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:32.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:32.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:32.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:32.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:32.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:32.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:32.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:32.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:32.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:32.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:32.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:32.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:32.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:32.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:32.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:48:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:48:32.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:48:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:32.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:34.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:34.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:34.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:34.267 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:34.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:34.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:34.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:34.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:34.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:34.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:34.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:34.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:34.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:34.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:34.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:34.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:34.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:34.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:34.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:48:34.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:48:34.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:48:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:34.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:48:38.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:48:38.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:38.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:38.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:38.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:38.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:38.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:38.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:38.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:38.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:48:38.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:38.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:38.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:38.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:38.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:38.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:48:38.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:48:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:48:38.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:38.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:38.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:39.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:37 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295311,"replay_state":"idle"} 2026-03-23T19:48:39.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:34 2026-03-23T19:48:39.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:48:39.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 185 .mirror.primary.0d382ca5-c95d-4b0a-9264-c4cd9429ae17.ba11ed90-d8a2-457c-8e64-2c81eecd25c9 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:48:39.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:48:39.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:48:39.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:48:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:39.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:39.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:39.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:39.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:39.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:37' 2026-03-23T19:48:39.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-03-23 19:48:37' 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:37' 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:39.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:39.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK257785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:26cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:34test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+stoppedlocal image is primary57845791mirror.1vm08.local2026-03-23 19:48:37cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295311,"replay_state":"idle"}2026-03-23 19:48:34' 2026-03-23T19:48:39.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:39.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:39.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:37' 2026-03-23T19:48:39.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:39.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:48:39.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:39.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:48:39.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:39.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:40.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d382ca5-c95d-4b0a-9264-c4cd9429ae17 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295311,"replay_state":"idle"} 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:34 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:37' 2026-03-23T19:48:40.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:48:40.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:48:40.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295311,"replay_state":"idle"} 2026-03-23T19:48:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:40.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:40.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:40.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:40.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:48:40.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:34' 2026-03-23T19:48:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-03-23 19:48:34' 2026-03-23T19:48:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:34' 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:40.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:40.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:39cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:26test_demote_image0d382ca5-c95d-4b0a-9264-c4cd9429ae17up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295311,"replay_state":"idle"}45194524mirror.1vm08.local2026-03-23 19:48:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:48:37' 2026-03-23T19:48:40.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:40.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:48:40.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:34' 2026-03-23T19:48:40.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:48:40.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:48:40.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:40.771 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:48:40.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:48:40.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_demote_image 2026-03-23T19:48:40.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:40.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:40.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:48:40.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-03-23T19:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-03-23T19:48:41.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:48:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:41.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:48:41.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: request image resync when remote is not primary' 2026-03-23T19:48:41.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:48:41.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:48:41.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:48:41 TEST: request image resync when remote is not primary 2026-03-23T19:48:41.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:48:41 TEST: request image resync when remote is not primary 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_resync_image=test_resync_image 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_resync_image 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_resync_image 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:48:41.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-03-23T19:48:41.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:48:41.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_resync_image': 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 17022c623998 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.17022c623998 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:48:41 2026 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:48:41 2026 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:48:41 2026 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:41.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:48:41.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:48:41.925 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:48:41.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:48:41.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 3030.3 bytes/sec: 12 MiB/s 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:41.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:42.967 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:42.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:42.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:42.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:42.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:42.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:42.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:42.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:42.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:43.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:43.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:43.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:43.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:43.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:43.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:43.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:43.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:43.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:43.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:43.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:48:43.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:48:43.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:43.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:43.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:43.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:43.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:43.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:48:43.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:48:43.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:43.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:44.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-03-23T19:48:44.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:44.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:44.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:44.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:44.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:44.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:48:44.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:44.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:48:44.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:44.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:44.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:48:44.208 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:44.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:44.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:44.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:48:44.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:44.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:44.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:48:44.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:44.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:44.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:44.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:48:44.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:48:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:48:44.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:44.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:44.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:48:44.400 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:48:44.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:44.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:44.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:48:44.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:44.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:44.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:48:44.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:48:44.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:44.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:44.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-03-23T19:48:44.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 182 2026-03-23T19:48:44.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-03-23T19:48:44.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:44.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:48:44.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-03-23T19:48:44.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-03-23T19:48:44.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-03-23T19:48:44.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:44.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:48:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=182 2026-03-23T19:48:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:48:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:44.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-03-23T19:48:45.004 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:45.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:45.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=181 2026-03-23T19:48:45.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 182 = 181 2026-03-23T19:48:45.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:48:45.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:48:45.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-03-23T19:48:45.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:48:45.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:48:45.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=182 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 182 = 182 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:45.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:46.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:46.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295321,"replay_state":"idle"} 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.3 on vm08.local 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:42 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:45' 2026-03-23T19:48:46.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:48:46.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:48:46.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295321,"replay_state":"idle"} 2026-03-23T19:48:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:46.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:46.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:46.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:46.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.3 on vm08.local 2026-03-23T19:48:46.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:46.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:42' 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-03-23 19:48:42' 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:42' 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:46.548 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:46.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK257785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:26cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:44test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295321,"replay_state":"idle"}57975803mirror.3vm08.local2026-03-23 19:48:42cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:45' 2026-03-23T19:48:46.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:46.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:46.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:42' 2026-03-23T19:48:46.601 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:46.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:48:46.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:46.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:46.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:47.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.2 on vm08.local 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:45 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295321,"replay_state":"idle"} 2026-03-23T19:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:42 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 181 .mirror.primary.0d588f72-8b2f-4510-a5ea-88ed52aad13e.12752dc9-0754-4fdc-b03f-ed6f223115dc (peer_uuids:[]) 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 182 .mirror.primary.0d588f72-8b2f-4510-a5ea-88ed52aad13e.cc704b13-f630-4c3f-9a5e-32fe95705f03 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:48:47.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:47.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:48:47.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:47.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:47.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:47.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:47.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:47.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.2 on vm08.local 2026-03-23T19:48:47.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:47.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:45' 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-03-23 19:48:45' 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:45' 2026-03-23T19:48:47.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:47.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:44cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:26test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+stoppedlocal image is primary45264532mirror.2vm08.local2026-03-23 19:48:45cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295321,"replay_state":"idle"}2026-03-23 19:48:42' 2026-03-23T19:48:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:47.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:45' 2026-03-23T19:48:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:47.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:48:47.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:47.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:47.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:48:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:48:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:48:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:48:47.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:48:47.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:48:47.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:48:47.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2500 bytes/sec: 9.8 MiB/s 2026-03-23T19:48:47.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test_resync_image 2026-03-23T19:48:47.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:47.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:47.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:47.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-03-23T19:48:49.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=test_resync_image_id 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_resync_image 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-03-23T19:48:49.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:48:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test_resync_image_id='\''16abae0f0bed'\''' 2026-03-23T19:48:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test_resync_image_id=16abae0f0bed 2026-03-23T19:48:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-03-23T19:48:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 16abae0f0bed 2026-03-23T19:48:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-03-23T19:48:49.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:49.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:50.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:48:50.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:50.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:50.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:50.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.2 on vm08.local 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:50 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:50.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:48:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:50' 2026-03-23T19:48:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:48:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:48:50.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:48:50.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:48:50.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:50.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:50.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:50.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:50.693 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:50.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.2 on vm08.local 2026-03-23T19:48:50.694 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:50.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:50' 2026-03-23T19:48:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-03-23 19:48:50' 2026-03-23T19:48:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:50' 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1145124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:49cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:26test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+unknownremote image is not primary45264532mirror.2vm08.local2026-03-23 19:48:50cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image demoted2026-03-23 19:48:50' 2026-03-23T19:48:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:50.736 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:50' 2026-03-23T19:48:50.737 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:48:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:50.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:48:50.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:50.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:51.742 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.3 on vm08.local 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:50 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:50' 2026-03-23T19:48:51.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:48:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:48:51.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:48:51.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:48:51.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:51.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:48:53.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:53.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.3 on vm08.local 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:52 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:50' 2026-03-23T19:48:53.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:48:53.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:48:53.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:48:53.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:48:53.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:53.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:53.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:53.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:53.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:53.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.3 on vm08.local 2026-03-23T19:48:53.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:53.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:52' 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-03-23 19:48:52' 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:52' 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:53.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:53.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1157785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:52cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:49test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+unknownremote image is not primary57975803mirror.3vm08.local2026-03-23 19:48:52cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:48:50' 2026-03-23T19:48:53.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:53.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:53.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:52' 2026-03-23T19:48:53.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:53.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:48:53.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:48:53.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_resync_image 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:48:53.894 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-03-23T19:48:54.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:54.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:48:55.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:48:55.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:48:55.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:48:55.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:48:55.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:48:55.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:55.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:55.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:48:55.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:55.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:55.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:55.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:48:55.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:48:55.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:55.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:55.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:48:55.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:48:55.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:48:55.402 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:48:55.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:48:55.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:48:55.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:48:55.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:55.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:55.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:48:55.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:48:55.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:48:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:48:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:48:55.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:55.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:56.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:56.481 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.3 on vm08.local 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:52 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"} 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:55 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 189 .mirror.primary.0d588f72-8b2f-4510-a5ea-88ed52aad13e.073046cd-28ba-4371-a4eb-f4ba9e994b74 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:48:56.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:48:56.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:48:56.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:56.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:48:58.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-03-23T19:48:58.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:58.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:58.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:58.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.3 on vm08.local 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:57 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"} 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:55 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 189 .mirror.primary.0d588f72-8b2f-4510-a5ea-88ed52aad13e.073046cd-28ba-4371-a4eb-f4ba9e994b74 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:48:58.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:48:58.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:58.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:48:58.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:58.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.3 on vm08.local 2026-03-23T19:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:57' 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-03-23 19:48:57' 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:57' 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK257785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:52cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:48:54test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+stoppedlocal image is primary57975803mirror.3vm08.local2026-03-23 19:48:57cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"}2026-03-23 19:48:55' 2026-03-23T19:48:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:58.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:58.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:57' 2026-03-23T19:48:58.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:58.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:48:58.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:48:58.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:48:58.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:48:58.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:48:59.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-03-23T19:48:59.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:59.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 0d588f72-8b2f-4510-a5ea-88ed52aad13e 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"} 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.2 on vm08.local 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:55 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:48:57' 2026-03-23T19:48:59.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:48:59.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:48:59.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:48:59.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"} 2026-03-23T19:48:59.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:48:59.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:59.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:48:59.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:59.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:48:59.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.2 on vm08.local 2026-03-23T19:48:59.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:48:59.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:48:59.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:55' 2026-03-23T19:48:59.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-03-23 19:48:55' 2026-03-23T19:48:59.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:48:55' 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:48:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:48:59cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:52test_resync_image0d588f72-8b2f-4510-a5ea-88ed52aad13eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295334,"replay_state":"idle"}45264532mirror.2vm08.local2026-03-23 19:48:55cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:48:57' 2026-03-23T19:48:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:48:59.941 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:48:59.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:48:55' 2026-03-23T19:48:59.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:48:59.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:48:59.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:48:59.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:48:59.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:48:59.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-03-23T19:48:59.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-03-23T19:48:59.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:59.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export 2026-03-23T19:48:59.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-03-23T19:48:59.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-03-23T19:48:59.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:48:59.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:48:59.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:48:59.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export 2026-03-23T19:49:00.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:49:00.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:00.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:00.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:00.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:00.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_resync_image 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:00.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-03-23T19:49:00.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-03-23T19:49:01.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:49:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:49:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: request image resync when remote is not primary and daemon is offline' 2026-03-23T19:49:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:49:01.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:49:01.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:49:01 TEST: request image resync when remote is not primary and daemon is offline 2026-03-23T19:49:01.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:49:01 TEST: request image resync when remote is not primary and daemon is offline 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_resync_image=test_resync_image 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_resync_image 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:01.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_resync_image 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:49:01.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-03-23T19:49:02.234 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:49:02.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_resync_image': 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 172fa9bbb831 2026-03-23T19:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.172fa9bbb831 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:49:01 2026 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:49:01 2026 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:49:01 2026 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:02.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:49:02.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:49:02.318 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:49:02.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:49:02.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2325.58 bytes/sec: 9.1 MiB/s 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:02.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:03.388 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:03.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:03.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:03.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:03.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:03.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:03.437 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:03.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:03.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:03.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:03.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:03.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:03.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:03.499 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:49:03.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:49:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:03.514 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:03.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:03.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:49:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:49:03.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:03.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:03.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:03.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:03.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:03.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:03.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:03.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:49:04.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:49:04.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:04.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:04.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:04.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:04.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:04.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:04.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:04.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:04.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:04.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:04.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:04.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:04.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:04.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:04.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:04.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:04.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:49:04.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:49:04.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:49:04.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:04.764 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:49:05.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:05.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:49:05.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:49:05.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:49:05.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:05.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:49:05.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:05.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:05.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:05.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:49:05.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:05.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:05.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:05.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:05.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:49:05.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:49:05.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:49:05.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:49:05.898 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:05.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-03-23T19:49:06.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 186 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-03-23T19:49:06.278 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:49:06.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:49:06.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-03-23T19:49:06.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-03-23T19:49:06.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.status 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-03-23T19:49:06.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:49:06.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:49:06.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=186 2026-03-23T19:49:06.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:49:06.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:49:06.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.status 2026-03-23T19:49:06.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:49:06.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-03-23T19:49:06.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=186 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 186 = 186 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:06.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:07.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295342,"replay_state":"idle"} 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:03 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:49:07.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:49:07.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:04' 2026-03-23T19:49:07.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:49:07.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:49:07.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:49:07.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295342,"replay_state":"idle"} 2026-03-23T19:49:07.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:07.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:49:07.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:07.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:49:07.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:07.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:07.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:07.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:03' 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-03-23 19:49:03' 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:03' 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:07.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:07.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK257785810mirror.0vm08.local20.2.0-712-g70f8415bfalseOK57845791mirror.1vm08.local20.2.0-712-g70f8415btrueOK57905796mirror.2vm08.local20.2.0-712-g70f8415bfalseOK57975803mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}57905796mirror.2vm08.local2026-03-23 19:48:52cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:49:04test_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295342,"replay_state":"idle"}57845791mirror.1vm08.local2026-03-23 19:49:03cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:49:04' 2026-03-23T19:49:07.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:07.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:07.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:03' 2026-03-23T19:49:07.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:07.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:49:07.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:49:07.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:07.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:08.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:08.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:04 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295342,"replay_state":"idle"} 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:03 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 185 .mirror.primary.b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a.ca419212-4a7e-4773-8bc6-da87c615f0c4 (peer_uuids:[]) 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 186 .mirror.primary.b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a.ae0709bb-dee8-49c1-ba36-d707558f1fcf (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:49:08.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:49:08.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:49:08.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:08.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:49:08.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:08.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:49:08.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:08.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:08.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:08.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:04' 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-03-23 19:49:04' 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:04' 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:08.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:08.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:49:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:48:52test_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+stoppedlocal image is primary45194524mirror.1vm08.local2026-03-23 19:49:04cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295342,"replay_state":"idle"}2026-03-23 19:49:03' 2026-03-23T19:49:08.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:08.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:08.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:04' 2026-03-23T19:49:08.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:08.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:49:08.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:49:08.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:49:08.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:08.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:08.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:08.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:49:08.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:08.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:08.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=94615 2026-03-23T19:49:08.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 94615 ']' 2026-03-23T19:49:08.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 94615 2026-03-23T19:49:08.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:08.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:08.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:08.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94615 '$2 == pid {print; exit 1}' 2026-03-23T19:49:08.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 94615 0.3 0.4 674828 46308 ? Ssl 19:48 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:08.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:08.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:09.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:09.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94615 '$2 == pid {print; exit 1}' 2026-03-23T19:49:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:09.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94615 '$2 == pid {print; exit 1}' 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:09.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:49:09.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:09.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:09.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:49:09.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:09.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:49:09.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:09.843 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:09.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:09.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:09.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:49:09.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:09.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:09.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=94650 2026-03-23T19:49:09.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 94650 ']' 2026-03-23T19:49:09.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 94650 2026-03-23T19:49:09.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:09.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:09.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:09.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94650 '$2 == pid {print; exit 1}' 2026-03-23T19:49:09.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 94650 0.5 0.5 737324 54192 ? Ssl 19:48 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:09.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:10.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:10.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94650 '$2 == pid {print; exit 1}' 2026-03-23T19:49:10.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:10.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:10.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94650 '$2 == pid {print; exit 1}' 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:49:10.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:10.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:10.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:49:10.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:10.944 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:10.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:49:10.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:10.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:10.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:49:10.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:49:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=94685 2026-03-23T19:49:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 94685 ']' 2026-03-23T19:49:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 94685 2026-03-23T19:49:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94685 '$2 == pid {print; exit 1}' 2026-03-23T19:49:10.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 94685 0.3 0.4 737324 49500 ? Ssl 19:48 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:10.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:10.994 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:11.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94685 '$2 == pid {print; exit 1}' 2026-03-23T19:49:12.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:12.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:12.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94685 '$2 == pid {print; exit 1}' 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:12.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:49:12.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:12.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:12.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:49:12.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:12.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:12.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:49:12.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:12.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:12.060 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:49:12.061 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:49:12.075 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:12.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=94745 2026-03-23T19:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 94745 ']' 2026-03-23T19:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 94745 2026-03-23T19:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:12.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:12.078 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94745 '$2 == pid {print; exit 1}' 2026-03-23T19:49:12.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 94745 0.4 0.5 737324 53524 ? Ssl 19:48 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:12.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:12.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:13.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94745 '$2 == pid {print; exit 1}' 2026-03-23T19:49:13.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:13.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=94745 '$2 == pid {print; exit 1}' 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:13.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:49:13.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:13.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:49:13.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:13.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:13.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:49:13.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:13.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:13.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-03-23T19:49:13.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:13.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:13.165 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:13.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:49:13.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:49:13.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:13.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:49:13.166 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:49:13.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:49:13.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:49:13.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2439.02 bytes/sec: 9.5 MiB/s 2026-03-23T19:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test_resync_image 2026-03-23T19:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-03-23T19:49:15.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:49:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-03-23T19:49:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:15.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=test_resync_image_id 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_resync_image 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-03-23T19:49:15.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:15.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test_resync_image_id='\''169fe6b7820b'\''' 2026-03-23T19:49:15.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test_resync_image_id=169fe6b7820b 2026-03-23T19:49:15.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-03-23T19:49:15.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 169fe6b7820b 2026-03-23T19:49:15.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-03-23T19:49:15.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:49:15.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:49:15.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:15.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:15.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:15.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:15.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:15.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:15.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:15.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:16.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:16.524 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:14 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:49:16.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:49:16.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:49:16.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:49:16.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:49:16.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:16.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:49:16.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:16.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:49:16.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:16.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:16.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:16.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:14' 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-03-23 19:49:14' 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:14' 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:49:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8fedown+unknownstatus not foundtest_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+unknownremote image is not primary45194524mirror.1vm08.local2026-03-23 19:49:14cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8fedown+unknownstatus not found' 2026-03-23T19:49:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:16.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:14' 2026-03-23T19:49:16.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:16.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:49:16.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:49:16.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:49:16.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:16.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:14' 2026-03-23T19:49:17.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:49:17.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:49:17.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:17.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:19.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:19.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:19.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295346,"remote_snapshot_timestamp":1774295354,"replay_state":"idle"} 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:19 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:14' 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:49:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:49:19.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:19.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:23.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image demoted 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:20 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:19' 2026-03-23T19:49:23.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:49:23.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:49:23.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:49:23.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:49:23.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:23.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 8 2026-03-23T19:49:31.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'description_pattern=remote image is not primary' 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:31.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:30 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:29' 2026-03-23T19:49:31.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:49:31.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:49:31.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*remote image is not primary' 2026-03-23T19:49:31.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:49:31.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:31.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:49:31.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:31.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:49:31.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:31.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:31.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:31.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:31.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:30' 2026-03-23T19:49:31.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-03-23 19:49:30' 2026-03-23T19:49:31.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:31.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:31.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:31.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:49:31.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:30' 2026-03-23T19:49:31.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:31.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:31.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:31.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING1159175921mirror.0vm08.local20.2.0-712-g70f8415btrueOK59235926mirror.1vm08.local20.2.0-712-g70f8415bfalseOK59325937mirror.2vm08.local20.2.0-712-g70f8415bfalseOK59425945mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}59175921mirror.0vm08.local2026-03-23 19:49:19cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:49:29test_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+unknownremote image is not primary59235926mirror.1vm08.local2026-03-23 19:49:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:49:29' 2026-03-23T19:49:31.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:31.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:31.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:30' 2026-03-23T19:49:31.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:49:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:49:31.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:49:31.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_resync_image 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:49:31.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-03-23T19:49:32.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:49:32.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-03-23T19:49:32.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:32.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:32.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:32.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:49:33.542 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:33.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:49:33.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:49:33.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:49:33.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:49:33.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:33.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:33.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:33.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:33.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:33.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:33.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:33.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:33.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:33.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:33.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:33.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:33.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:33.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:33.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:33.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:49:33.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:49:33.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:49:33.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:49:33.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:49:33.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:33.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:35.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:49:35.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:49:35.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:49:35.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:35.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:35.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:35.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:35.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:35.733 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:35.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:35.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-03-23T19:49:35.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:49:35.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:35.791 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:35.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:35.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:35.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:35.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:35.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-03-23T19:49:35.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:49:35.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:49:35.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:49:35.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:35.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:35.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:36.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-03-23T19:49:36.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:36.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:35 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295372,"replay_state":"idle"} 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:34 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 193 .mirror.primary.b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a.3d80d6a4-57b2-4f97-92ce-add7140338d4 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:49:36.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:49:36.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:49:36.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:36.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:49:36.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:36.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:49:36.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:36.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:36.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:36.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:35' 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-03-23 19:49:35' 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:35' 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:36.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:36.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK259175921mirror.0vm08.local20.2.0-712-g70f8415btrueOK59235926mirror.1vm08.local20.2.0-712-g70f8415bfalseOK59325937mirror.2vm08.local20.2.0-712-g70f8415bfalseOK59425945mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}59175921mirror.0vm08.local2026-03-23 19:49:19cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:49:34test_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+stoppedlocal image is primary59235926mirror.1vm08.local2026-03-23 19:49:35cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295372,"replay_state":"idle"}2026-03-23 19:49:34' 2026-03-23T19:49:36.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:36.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:36.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:35' 2026-03-23T19:49:36.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:36.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:49:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:49:36.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:36.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:49:36.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:37.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:37.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-03-23T19:49:38.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_resync_image: 2026-03-23T19:49:38.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b7ccabf1-3721-4e11-b6f4-d9e28c3ed05a 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295372,"replay_state":"idle"} 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:34 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:49:35' 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295372,"replay_state":"idle"} 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:49:38.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:34' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-03-23 19:49:34' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:49:34' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK245124515mirror.0vm08.local20.2.0-712-g70f8415btrueOK45194524mirror.1vm08.local20.2.0-712-g70f8415bfalseOK45264532mirror.2vm08.local20.2.0-712-g70f8415bfalseOK45364545mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest4e71904d-0696-4b3f-a95f-dcae5da3a2cfup+stoppedlocal image is primary45124515mirror.0vm08.local2026-03-23 19:49:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1774295157,"remote_snapshot_timestamp":1774295157,"replay_state":"idle"}2026-03-23 19:49:19test_resync_imageb7ccabf1-3721-4e11-b6f4-d9e28c3ed05aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295372,"replay_state":"idle"}45194524mirror.1vm08.local2026-03-23 19:49:34cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:49:35' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:49:34' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:49:38.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_resync_image 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export 2026-03-23T19:49:38.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:49:38.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:49:38.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_resync_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_resync_image.export 2026-03-23T19:49:38.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:38.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-03-23T19:49:38.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:38.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_resync_image 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_resync_image 2026-03-23T19:49:38.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-03-23T19:49:39.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-03-23T19:49:40.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:49:40.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:40.180 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: disable mirror while daemon is stopped' 2026-03-23T19:49:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:49:40.181 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:49:40 TEST: disable mirror while daemon is stopped 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:49:40 TEST: disable mirror while daemon is stopped 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:40.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:40.183 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:40.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:49:40.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:40.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:40.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=96843 2026-03-23T19:49:40.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 96843 ']' 2026-03-23T19:49:40.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 96843 2026-03-23T19:49:40.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:40.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:40.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:40.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96843 '$2 == pid {print; exit 1}' 2026-03-23T19:49:40.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 96843 0.5 0.4 737324 48368 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:40.222 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:40.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:41.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:41.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96843 '$2 == pid {print; exit 1}' 2026-03-23T19:49:41.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:41.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:41.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96843 '$2 == pid {print; exit 1}' 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:41.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:49:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:41.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:41.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:41.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:49:41.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:41.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:41.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:41.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:49:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:41.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:41.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:41.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:49:41.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:41.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:41.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=96887 2026-03-23T19:49:41.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 96887 ']' 2026-03-23T19:49:41.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 96887 2026-03-23T19:49:41.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:41.314 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:41.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:41.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96887 '$2 == pid {print; exit 1}' 2026-03-23T19:49:41.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 96887 0.5 0.5 737324 50096 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:41.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:41.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:42.331 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:42.332 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96887 '$2 == pid {print; exit 1}' 2026-03-23T19:49:42.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:42.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:42.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96887 '$2 == pid {print; exit 1}' 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:42.351 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:49:42.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:42.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:42.367 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:49:42.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:42.398 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:42.399 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:49:42.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:42.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:42.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:49:42.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:42.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:42.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=96963 2026-03-23T19:49:42.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 96963 ']' 2026-03-23T19:49:42.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 96963 2026-03-23T19:49:42.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:42.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:42.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:42.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96963 '$2 == pid {print; exit 1}' 2026-03-23T19:49:42.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 96963 0.4 0.4 674832 44768 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:42.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:42.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:43.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:43.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96963 '$2 == pid {print; exit 1}' 2026-03-23T19:49:43.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:43.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:43.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=96963 '$2 == pid {print; exit 1}' 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:43.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:49:43.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:43.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:43.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:43.493 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:49:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:43.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:43.511 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:49:43.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:43.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:43.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:49:43.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:49:43.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:43.545 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:43.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=97049 2026-03-23T19:49:43.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 97049 ']' 2026-03-23T19:49:43.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 97049 2026-03-23T19:49:43.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:43.546 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:43.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:43.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97049 '$2 == pid {print; exit 1}' 2026-03-23T19:49:43.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 97049 0.4 0.4 671760 45880 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:43.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:43.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:44.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:44.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97049 '$2 == pid {print; exit 1}' 2026-03-23T19:49:44.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:44.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:44.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97049 '$2 == pid {print; exit 1}' 2026-03-23T19:49:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:49:44.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:44.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:49:44.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:44.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:44.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:44.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:49:44.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:44.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:44.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:49:44.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:44.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:49:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster2 2026-03-23T19:49:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:44.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:0 '' 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:0 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:44.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:49:44.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-03-23T19:49:44.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:49:44.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:49:44.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=71541 2026-03-23T19:49:44.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 71541 ']' 2026-03-23T19:49:44.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 71541 2026-03-23T19:49:44.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:44.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:44.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:44.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71541 '$2 == pid {print; exit 1}' 2026-03-23T19:49:44.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 71541 0.3 0.6 746544 67084 ? Ssl 19:34 0:02 rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:44.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:44.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:45.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:45.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71541 '$2 == pid {print; exit 1}' 2026-03-23T19:49:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:45.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71541 '$2 == pid {print; exit 1}' 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:0 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:45.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:45.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:45.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:0 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:45.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:49:45.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:45.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:49:45.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-03-23T19:49:45.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:49:45.765 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:1 '' 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:1 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:45.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:49:45.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-03-23T19:49:45.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:49:45.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:49:45.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=71585 2026-03-23T19:49:45.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 71585 ']' 2026-03-23T19:49:45.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 71585 2026-03-23T19:49:45.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:45.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:45.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:45.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71585 '$2 == pid {print; exit 1}' 2026-03-23T19:49:45.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 71585 0.1 0.6 540624 66136 ? Ssl 19:34 0:01 rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:45.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:45.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:46.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71585 '$2 == pid {print; exit 1}' 2026-03-23T19:49:46.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:46.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:46.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71585 '$2 == pid {print; exit 1}' 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:46.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:46.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:46.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:1 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:46.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:46.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:46.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:46.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:46.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:46.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:49:46.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:46.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:49:46.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-03-23T19:49:46.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:49:46.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:49:46.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:2 '' 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:49:46.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:49:46.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-03-23T19:49:46.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:49:46.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:49:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=71646 2026-03-23T19:49:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 71646 ']' 2026-03-23T19:49:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 71646 2026-03-23T19:49:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:46.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:46.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:46.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71646 '$2 == pid {print; exit 1}' 2026-03-23T19:49:46.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 71646 0.1 0.6 532424 65360 ? Ssl 19:34 0:01 rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:46.924 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:47.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:47.926 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71646 '$2 == pid {print; exit 1}' 2026-03-23T19:49:47.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:47.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:47.935 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71646 '$2 == pid {print; exit 1}' 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:2 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:47.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:47.946 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:49:47.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:47.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:47.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:49:47.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:47.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:47.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:49:47.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:49:47.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-03-23T19:49:47.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:49:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:3 '' 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:3 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:49:47.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:49:47.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-03-23T19:49:48.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:49:48.011 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=71730 2026-03-23T19:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 71730 ']' 2026-03-23T19:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 71730 2026-03-23T19:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:49:48.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71730 '$2 == pid {print; exit 1}' 2026-03-23T19:49:48.021 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:48.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 71730 0.1 0.5 311136 58396 ? Rsl 19:34 0:01 rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:48.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:49:48.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:49.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:49.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71730 '$2 == pid {print; exit 1}' 2026-03-23T19:49:49.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:49:49.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:49:49.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=71730 '$2 == pid {print; exit 1}' 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:3 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:49.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:49:49.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:49.082 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:3 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:49:49.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:49.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:49:49.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-03-23T19:49:49.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:49:49.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ disable_mirror cluster2 mirror test 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:49.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test 2026-03-23T19:49:49.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring disabled 2026-03-23T19:49:49.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:49.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1137a92354c0 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1137a92354c0 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z '' -o '' = 1137a92354c0 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:49.452 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:49.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:49.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:49.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:49:49.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:49:49.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:49:49.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:49:49.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:49.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:49:49.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:49.600 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:49.612 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:49:49.609+0000 7f9862b48640 -1 reset not still connected to 0x55d6c1a74ea0 2026-03-23T19:49:49.612 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:49:49.609+0000 7f9862b48640 -1 reset not still connected to 0x55d6c1a752b0 2026-03-23T19:49:49.612 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:49:49.609+0000 7f9862b48640 -1 reset not still connected to 0x55d6c2c52b60 2026-03-23T19:49:49.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=1137a92354c0 2026-03-23T19:49:49.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:49.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 1137a92354c0 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1137a92354c0 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:49.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:49.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1137a92354c0 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1137a92354c0 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1137a92354c0 -o 1137a92354c0 = 1137a92354c0 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:49.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:50.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 1137a92354c0 2026-03-23T19:49:50.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:50.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:50.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:50.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1137a92354c0 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:50.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1137a92354c0 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1137a92354c0 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1137a92354c0 -o 1137a92354c0 = 1137a92354c0 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:50.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 1137a92354c0 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1137a92354c0 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:52.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:52.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:52.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:52.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1137a92354c0 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1137a92354c0 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1137a92354c0 -o 1137a92354c0 = 1137a92354c0 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:52.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:49:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 1137a92354c0 2026-03-23T19:49:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1137a92354c0 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:56.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:56.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:49:56.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:49:56.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:56.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:49:56.936 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:49:56.937 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-03-23T19:49:57.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:49:57.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 118eeabff75e 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.118eeabff75e 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:34:13 2026 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:35:21 2026 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:35:31 2026 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: cea13f2e-b2ae-4f72-b25b-171ed35679d2 2026-03-23T19:49:57.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test present 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:57.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:57.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:57.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:57.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:49:57.734 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:49:57.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:57.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:57.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:57.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = deleted 2026-03-23T19:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:49:58.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:49:58.896 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:49:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:49:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:49:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = deleted 2026-03-23T19:49:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:49:58.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:50:00.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:50:00.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:00.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176dfb92c7de 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176dfb92c7de 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z '' -o '' = 176dfb92c7de 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:00.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:01.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:01.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:01.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:01.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:01.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:01.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:50:02.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:02.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:02.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:02.038 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: non-default namespace image mirroring' 2026-03-23T19:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:02 TEST: non-default namespace image mirroring 2026-03-23T19:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:02 TEST: non-default namespace image mirroring 2026-03-23T19:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - replay' 2026-03-23T19:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:02.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:02.042 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:02 - replay 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:02 - replay 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns1 test 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns1/test 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:02.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-03-23T19:50:02.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:50:02.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1785c9709ba4 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1785c9709ba4 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: b228aa89-9a2d-443e-9b79-37a1542517fe 2026-03-23T19:50:02.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns2 test snapshot 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns2 test 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:02.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns2/test 2026-03-23T19:50:02.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns2 2026-03-23T19:50:02.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:02.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns2 test snapshot 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:02.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns2/test snapshot 2026-03-23T19:50:03.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:50:03.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns2/test 2026-03-23T19:50:03.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:50:03.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:50:03.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:50:03.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:50:03.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 178e7a8e7b75 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.178e7a8e7b75 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:50:02 2026 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 45c552f7-4896-4a63-b6eb-694413fd2297 2026-03-23T19:50:03.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:03.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:04.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:04.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:04.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:04.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:04.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:04.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:04.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:04.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:04.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:04.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:04.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-03-23T19:50:04.878 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:04.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:04.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:04.879 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:04.880 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns2 test 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns2 test started 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:04.881 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:05.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns2 test started 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns2/test 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:05.883 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:05.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:05.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns2/test 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:05.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:05.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:05.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns2/test 2026-03-23T19:50:05.958 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:05.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:05.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:05.959 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:05.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:05.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:05.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:05.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:50:05.961 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:50:06.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:50:06.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:50:06.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2439.02 bytes/sec: 9.5 MiB/s 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns2 test 100 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:50:06.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns2/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:50:06.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:50:06.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:50:06.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1515.15 bytes/sec: 5.9 MiB/s 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:06.164 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-03-23T19:50:06.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 192 2026-03-23T19:50:06.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-03-23T19:50:06.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test.status 2026-03-23T19:50:06.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:06.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:50:06.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-03-23T19:50:06.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test.status 2026-03-23T19:50:06.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.status 2026-03-23T19:50:06.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-03-23T19:50:06.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:06.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:50:06.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=192 2026-03-23T19:50:06.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:50:06.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:06.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-03-23T19:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:07.114 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:07.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=190 2026-03-23T19:50:07.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 192 = 190 2026-03-23T19:50:07.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:07.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.status 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-03-23T19:50:07.518 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:07.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:07.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=192 2026-03-23T19:50:07.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 192 = 192 2026-03-23T19:50:07.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns2 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns2 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns2 test 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns2/test 2026-03-23T19:50:07.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 193 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns2 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns2 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns2-test.status 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns2-test.status 2026-03-23T19:50:07.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:07.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.status 2026-03-23T19:50:07.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns2-test.status 2026-03-23T19:50:07.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns2-test.status 2026-03-23T19:50:07.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.status 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns2 test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.status 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:07.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.status 2026-03-23T19:50:07.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns2/test --format xml 2026-03-23T19:50:07.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:07.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:50:07.862 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=193 2026-03-23T19:50:07.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:50:07.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:07.863 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.status 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.status 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-03-23T19:50:08.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:08.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=191 2026-03-23T19:50:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 193 = 191 2026-03-23T19:50:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:08.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.status 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.status 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-03-23T19:50:08.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:08.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=193 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 193 = 193 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:08.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:08.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:08.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:08.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:08.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:08.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:09.570 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: b228aa89-9a2d-443e-9b79-37a1542517fe 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295402,"replay_state":"idle"} 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:04 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:09.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:09.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:09.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:09.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295402,"replay_state":"idle"} 2026-03-23T19:50:09.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:09.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:09.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:09.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:09.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:09.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:04' 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-03-23 19:50:04' 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:04' 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:09.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-03-23T19:50:09.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestb228aa89-9a2d-443e-9b79-37a1542517feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295402,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:04cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:09.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:09.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:09.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:04' 2026-03-23T19:50:09.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:09.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:09.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:09.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:09.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns2 test 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns2 test up+replaying 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:09.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns2 test up+replaying '' '' 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:10.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:10.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:10.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns2/test 2026-03-23T19:50:10.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 45c552f7-4896-4a63-b6eb-694413fd2297 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295403,"replay_state":"idle"} 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:05 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:10.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:10.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:10.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295403,"replay_state":"idle"} 2026-03-23T19:50:10.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:10.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:10.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:10.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:10.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:10.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:10.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:05' 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns2 test up+replaying '2026-03-23 19:50:05' 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:10.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:10.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:05' 2026-03-23T19:50:10.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:10.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:10.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns2 --verbose --format xml 2026-03-23T19:50:10.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest45c552f7-4896-4a63-b6eb-694413fd2297up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295403,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:05cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:10.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:10.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:10.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:05' 2026-03-23T19:50:10.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:10.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:10.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:10.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:10.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-03-23T19:50:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test.export 2026-03-23T19:50:10.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export 2026-03-23T19:50:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-03-23T19:50:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test.export 2026-03-23T19:50:10.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:50:10.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:50:10.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export 2026-03-23T19:50:11.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:11.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:50:11.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:11.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:50:11.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test.export 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns2 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns2 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns2-test.export 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns2-test.export 2026-03-23T19:50:11.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:11.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.export 2026-03-23T19:50:11.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns2-test.export 2026-03-23T19:50:11.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns2-test.export 2026-03-23T19:50:11.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:11.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.export 2026-03-23T19:50:11.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.export 2026-03-23T19:50:11.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns2/test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.export 2026-03-23T19:50:11.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:11.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns2/test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.export 2026-03-23T19:50:11.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:11.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.export 2026-03-23T19:50:12.007 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns2-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns2-test.export 2026-03-23T19:50:12.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:12.025 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - disable mirroring / delete image' 2026-03-23T19:50:12.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:12.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:12.026 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:12 - disable mirroring / delete image 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:12 - disable mirroring / delete image 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:50:12.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns1 test 2026-03-23T19:50:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:12.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-03-23T19:50:12.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-03-23T19:50:13.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ disable_mirror cluster2 mirror/ns2 test 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:13.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/ns2/test 2026-03-23T19:50:14.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring disabled 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:14.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-03-23T19:50:14.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:14.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:14.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:50:14.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:14.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:14.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-03-23T19:50:14.395 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:14.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:14.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror/ns2 test deleted 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns2 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-03-23T19:50:14.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:14.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:14.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:50:14.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:14.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror/ns2 test deleted '' 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns2 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-03-23T19:50:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:14.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns2 test 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:50:14.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:14.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns2 test 2026-03-23T19:50:14.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:14.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns2 2026-03-23T19:50:14.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns2/test 2026-03-23T19:50:14.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns2/test 2026-03-23T19:50:14.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:50:14.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:14.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: mirror to a different remote namespace' 2026-03-23T19:50:14.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:14.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:14.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:14 TEST: mirror to a different remote namespace 2026-03-23T19:50:14.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:14 TEST: mirror to a different remote namespace 2026-03-23T19:50:14.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - replay' 2026-03-23T19:50:14.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:14.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:14.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:14 - replay 2026-03-23T19:50:14.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:14 - replay 2026-03-23T19:50:14.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ NS3=ns3 2026-03-23T19:50:14.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ NS4=ns4 2026-03-23T19:50:14.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 namespace create mirror/ns3 2026-03-23T19:50:14.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 namespace create mirror/ns4 2026-03-23T19:50:14.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns3 image --remote-namespace ns4 2026-03-23T19:50:14.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns4 image --remote-namespace ns3 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns4 test snapshot 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns4 test 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:14.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns4/test 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns4 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns4 test snapshot 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:14.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns4/test snapshot 2026-03-23T19:50:15.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:50:15.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns4/test 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 17c01fc9a6d5 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.17c01fc9a6d5 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:50:14 2026 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:50:14 2026 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:50:14 2026 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: f0cd42f8-f141-4384-a0f4-970743032bf5 2026-03-23T19:50:15.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns3 test 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns3 test started 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:15.294 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns3 test started 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:16.295 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:16.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:16.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:16.299 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:16.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:16.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:16.361 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:16.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:16.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:16.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:16.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:16.429 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:16.430 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:16.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:16.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:16.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:16.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:16.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:16.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:16.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:16.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:16.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:16.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:16.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:16.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:16.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:16.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:50:17.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:17.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:17.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:17.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:17.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:17.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:17.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:17.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:17.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:17.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:17.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:17.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:17.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:17.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:17.626 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:17.627 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:17.641 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:17.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:17.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:17.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:17.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:17.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:17.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:17.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:17.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:17.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:17.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:17.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:17.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:17.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:17.758 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:17.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:17.773 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:17.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:17.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:17.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:50:19.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:19.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:19.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:19.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:19.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:19.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:19.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:19.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:19.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:19.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:19.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:19.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:19.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:19.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:19.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:19.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:19.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:19.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:19.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:19.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:19.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:19.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:19.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:19.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:20.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:20.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:20.018 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:20.020 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:20.036 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:20.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:20.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:20.037 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:20.083 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:20.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:20.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:20.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:20.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 4 2026-03-23T19:50:24.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:24.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:24.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:24.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:24.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:24.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:24.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:24.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:24.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:24.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:24.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:24.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:24.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:24.161 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns4 test 100 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:50:24.162 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns4/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:50:24.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:50:24.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:50:24.241 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2564.1 bytes/sec: 10 MiB/s 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns4 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns4 test 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns4/test 2026-03-23T19:50:24.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 195 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns4 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns4-test.status 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns4-test.status 2026-03-23T19:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:24.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.status 2026-03-23T19:50:24.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns3-test.status 2026-03-23T19:50:24.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns3-test.status 2026-03-23T19:50:24.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns4 test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.status 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.status 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns4/test --format xml 2026-03-23T19:50:24.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:24.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:50:24.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=195 2026-03-23T19:50:24.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:50:24.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:24.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-03-23T19:50:24.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:24.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=195 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 195 = 195 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns3 test 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:24.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns3 test up+replaying 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:24.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns3 test up+replaying '' '' 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:25.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns3/test 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: f0cd42f8-f141-4384-a0f4-970743032bf5 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295415,"replay_state":"idle"} 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:21 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:25.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:25.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:25.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:25.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295415,"replay_state":"idle"} 2026-03-23T19:50:25.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:25.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:25.688 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:25.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:25.689 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:25.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:25.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:25.690 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:21' 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns3 test up+replaying '2026-03-23 19:50:21' 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:21' 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:25.692 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns3 --verbose --format xml 2026-03-23T19:50:25.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestf0cd42f8-f141-4384-a0f4-970743032bf5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295415,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:21cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:25.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:25.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:25.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:21' 2026-03-23T19:50:25.743 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:25.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:25.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:25.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:25.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:25.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:25.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:25.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns4 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns4-test.export 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns4-test.export 2026-03-23T19:50:25.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:25.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.export 2026-03-23T19:50:25.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns3-test.export 2026-03-23T19:50:25.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns3-test.export 2026-03-23T19:50:25.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:25.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:25.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:25.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns4/test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.export 2026-03-23T19:50:25.876 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:25.882 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns3/test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:26.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:26.013 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:26.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns4-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:26.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:26.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - disable mirroring and re-enable without remote-namespace' 2026-03-23T19:50:26.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:26.084 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:26.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:26 - disable mirroring and re-enable without remote-namespace 2026-03-23T19:50:26.085 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:26 - disable mirroring and re-enable without remote-namespace 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns4 test 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:50:26.086 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns4 test 2026-03-23T19:50:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns4 2026-03-23T19:50:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:26.087 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns4/test 2026-03-23T19:50:26.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns4/test 2026-03-23T19:50:26.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror/ns3 test deleted 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns3 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:26.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-03-23T19:50:26.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:26.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:26.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:50:26.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:26.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror/ns3 test deleted '' 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns3 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:26.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-03-23T19:50:26.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:26.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:26.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:26.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:26.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:26.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns3 2026-03-23T19:50:26.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns4 2026-03-23T19:50:27.012 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 namespace create mirror/ns3 2026-03-23T19:50:27.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns3 image 2026-03-23T19:50:27.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns3 image 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns3 test snapshot 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:27.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns3 test 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:27.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns3/test 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns3 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns3 test snapshot 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:27.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns3/test snapshot 2026-03-23T19:50:27.739 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:50:27.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns3/test 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test': 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 17e14570a083 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.17e14570a083 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:50:27 2026 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:50:27 2026 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:50:27 2026 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 29bd7c52-f526-497d-821e-c6e50eeb424e 2026-03-23T19:50:27.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:50:27.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns3 test 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns3 test started 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:27.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns3 test started 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:28.782 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:28.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:28.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:28.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:28.783 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:28.784 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:28.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:28.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:28.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:28.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:28.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:28.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:28.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:28.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:28.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:28.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:28.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:28.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:28.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:28.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:28.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:28.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:28.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:28.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:28.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:28.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:28.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:28.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:28.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:28.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:28.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:28.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:28.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:28.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:28.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:28.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:28.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:28.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:28.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:29.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:29.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:29.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:29.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:29.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:50:30.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:30.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:30.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:30.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:30.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:30.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:30.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:30.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:30.067 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:30.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:30.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:30.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:30.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:30.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:30.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:30.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:30.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:30.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:30.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:30.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:30.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:30.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:30.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:30.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:30.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:30.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:30.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:30.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:30.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:30.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:30.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:30.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:30.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-03-23T19:50:30.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:30.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:30.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:30.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:50:32.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:32.328 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:32.329 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:32.330 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:32.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:32.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:32.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:32.352 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-03-23T19:50:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:32.410 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:32.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:32.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:32.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:32.411 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns3 test 100 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:50:32.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns3/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:50:32.453 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:50:32.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:50:32.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1886.79 bytes/sec: 7.4 MiB/s 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns3 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns3 test 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:32.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns3/test 2026-03-23T19:50:32.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 197 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns3 2026-03-23T19:50:32.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:32.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns3-test.status 2026-03-23T19:50:32.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns3-test.status 2026-03-23T19:50:32.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:32.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.status 2026-03-23T19:50:32.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns3-test.status 2026-03-23T19:50:32.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns3-test.status 2026-03-23T19:50:32.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns3 test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.status 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:32.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.status 2026-03-23T19:50:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns3/test --format xml 2026-03-23T19:50:32.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:32.945 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:50:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=197 2026-03-23T19:50:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:50:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:32.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:50:33.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-03-23T19:50:33.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:33.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:33.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=196 2026-03-23T19:50:33.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 197 = 196 2026-03-23T19:50:33.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:33.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-03-23T19:50:33.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:33.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:33.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=196 2026-03-23T19:50:33.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 197 = 196 2026-03-23T19:50:33.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:33.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.status 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-03-23T19:50:34.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:34.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=197 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 197 = 197 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns3 test 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:34.473 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns3 test up+replaying 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:34.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns3 test up+replaying '' '' 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:35.475 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns3/test 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 29bd7c52-f526-497d-821e-c6e50eeb424e 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295427,"replay_state":"idle"} 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:31 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:35.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:35.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295427,"replay_state":"idle"} 2026-03-23T19:50:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:35.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:35.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:35.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:35.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:35.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:31' 2026-03-23T19:50:35.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns3 test up+replaying '2026-03-23 19:50:31' 2026-03-23T19:50:35.536 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:31' 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:35.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns3 --verbose --format xml 2026-03-23T19:50:35.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest29bd7c52-f526-497d-821e-c6e50eeb424eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295427,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:31cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:35.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:35.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:31' 2026-03-23T19:50:35.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:35.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:35.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:35.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:35.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:35.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:35.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:35.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns3 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns3 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns3-test.export 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns3-test.export 2026-03-23T19:50:35.582 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:35.583 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.export 2026-03-23T19:50:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns3-test.export 2026-03-23T19:50:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns3-test.export 2026-03-23T19:50:35.584 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:35.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:35.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:35.586 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns3/test /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.export 2026-03-23T19:50:35.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:35.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns3/test /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:35.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:36.034 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns3-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns3-test.export 2026-03-23T19:50:36.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns3 test 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns3 test 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:36.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns3/test 2026-03-23T19:50:36.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns3/test 2026-03-23T19:50:37.322 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror/ns3 test deleted 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns3 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-03-23T19:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:37.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:50:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:37.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror/ns3 test deleted '' 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns3 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns3 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-03-23T19:50:37.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:37.492 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:37.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:37.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:37.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:37.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:37.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns3 2026-03-23T19:50:37.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns3 2026-03-23T19:50:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog ' - data pool' 2026-03-23T19:50:37.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:37.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:37.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:37 - data pool 2026-03-23T19:50:37.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:37 - data pool 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ dp_image=test_data_pool 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 128 --data-pool mirror_parent 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror/ns1 test_data_pool 128 --data-pool mirror_parent 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:37.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 128 ']' 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=128 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:37.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock --data-pool mirror_parent mirror/ns1/test_data_pool 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:50:37.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test_data_pool snapshot 2026-03-23T19:50:38.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:50:38.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test_data_pool 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_data_pool': 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 17fe696657dd 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: data_pool: mirror_parent 2026-03-23T19:50:38.319 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.3.17fe696657dd 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, data-pool 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:50:37 2026 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:50:37 2026 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:50:37 2026 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 4761259d-d10d-40f0-b6c6-c147f78337e2 2026-03-23T19:50:38.320 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_data_pool cluster2 mirror/ns1 test_data_pool 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_data_pool 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 info mirror/ns1/test_data_pool 2026-03-23T19:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ data_pool=mirror_parent 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test mirror_parent = mirror_parent 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test_data_pool 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:39.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:39.363 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:39.364 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:39.365 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:39.381 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-03-23T19:50:39.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:39.431 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:39.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:50:39.435 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:50:39.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:50:39.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-03-23T19:50:39.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:39.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:50:39.507 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:50:39.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:50:39.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:39.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:39.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:50:39.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-03-23T19:50:39.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:39.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:39.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:50:39.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:50:39.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:39.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:39.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:50:39.595 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:39.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-03-23T19:50:39.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:50:39.644 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:50:39.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:39.645 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:50:40.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:40.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:40.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:40.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:40.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-03-23T19:50:40.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:40.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:40.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:40.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_data_pool cluster1 mirror/ns1 test_data_pool 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror/ns1 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_data_pool 2026-03-23T19:50:40.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 info mirror/ns1/test_data_pool 2026-03-23T19:50:40.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ data_pool=mirror_parent 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test mirror_parent = mirror_parent 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror/ns1 test_data_pool 100 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:50:40.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:50:40.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:50:40.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:50:40.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2325.58 bytes/sec: 9.1 MiB/s 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test_data_pool 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:40.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test_data_pool 2026-03-23T19:50:41.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 17 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test_data_pool.status 2026-03-23T19:50:41.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:41.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.status 2026-03-23T19:50:41.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test_data_pool.status 2026-03-23T19:50:41.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test_data_pool /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test_data_pool --format xml 2026-03-23T19:50:41.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:41.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:50:41.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=17 2026-03-23T19:50:41.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:50:41.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:41.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.550 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-03-23T19:50:41.551 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:41.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:41.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=16 2026-03-23T19:50:41.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 17 = 16 2026-03-23T19:50:41.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:41.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-03-23T19:50:41.990 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:42.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=16 2026-03-23T19:50:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 17 = 16 2026-03-23T19:50:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:50:42.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:50:42.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.status 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-03-23T19:50:42.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:50:42.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=17 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 17 = 17 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test_data_pool 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:42.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying '' '' 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:43.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:43.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test_data_pool 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_data_pool: 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 4761259d-d10d-40f0-b6c6-c147f78337e2 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295438,"replay_state":"idle"} 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:40 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:43.915 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:43.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:43.916 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:43.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295438,"replay_state":"idle"} 2026-03-23T19:50:43.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:43.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:43.918 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:43.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:43.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:43.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:43.920 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:43.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:43.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:40' 2026-03-23T19:50:43.922 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test_data_pool up+replaying '2026-03-23 19:50:40' 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:40' 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:43.923 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-03-23T19:50:43.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_data_pool4761259d-d10d-40f0-b6c6-c147f78337e2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295438,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:43.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:43.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-03-23T19:50:43.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:40' 2026-03-23T19:50:43.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-03-23T19:50:43.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:43.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:43.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror/ns1 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror/ns1 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.export 2026-03-23T19:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror/ns1-test_data_pool.export 2026-03-23T19:50:43.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.export 2026-03-23T19:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.export 2026-03-23T19:50:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror/ns1-test_data_pool.export 2026-03-23T19:50:43.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:43.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.export 2026-03-23T19:50:43.974 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.export 2026-03-23T19:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/ns1/test_data_pool /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.export 2026-03-23T19:50:44.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:44.111 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/ns1/test_data_pool /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.export 2026-03-23T19:50:44.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:44.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.export 2026-03-23T19:50:44.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.ZXt03XlClS/cluster1-mirror_ns1-test_data_pool.export 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror/ns1 test_data_pool 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:50:44.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:44.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror/ns1 test_data_pool 2026-03-23T19:50:44.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:44.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror/ns1 2026-03-23T19:50:44.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_data_pool 2026-03-23T19:50:44.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test_data_pool 2026-03-23T19:50:44.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test_data_pool 2026-03-23T19:50:45.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:50:45.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:45.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: simple image resync' 2026-03-23T19:50:45.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:45.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:45.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:45 TEST: simple image resync 2026-03-23T19:50:45.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:45 TEST: simple image resync 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror test image_id 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=image_id 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:45.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:45.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'image_id='\''176dfb92c7de'\''' 2026-03-23T19:50:45.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ image_id=176dfb92c7de 2026-03-23T19:50:45.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$image_id"' 2026-03-23T19:50:45.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 176dfb92c7de 2026-03-23T19:50:45.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-03-23T19:50:45.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 176dfb92c7de 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176dfb92c7de 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176dfb92c7de 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176dfb92c7de 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:45.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176dfb92c7de 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:45.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test present 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:45.988 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:45.989 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:46.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id= 2026-03-23T19:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:46.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:46.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:46.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:46.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:46.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = deleted 2026-03-23T19:50:46.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:46.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:47.159 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:47.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:47.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:47.189 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:47.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:47.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:47.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = deleted 2026-03-23T19:50:47.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:47.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:49.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:49.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = deleted 2026-03-23T19:50:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:49.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present '' 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:53.230 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176d65776e8f 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d65776e8f 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z '' -o '' = 176d65776e8f 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:53.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:54.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:54.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:54.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:54.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:54.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:54.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:54.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:54.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:54.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:54.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:50:54.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:54.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:54.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:54.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:54.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:54.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:54.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:54.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:50:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:55.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:55.347 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:50:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: cea13f2e-b2ae-4f72-b25b-171ed35679d2 2026-03-23T19:50:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:55.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:49 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:55.390 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:50:55.391 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:55.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:55.392 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:55.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:55.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:55.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:49' 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:50:49' 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:49' 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:55.396 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:50:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestcea13f2e-b2ae-4f72-b25b-171ed35679d2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:49cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:55.439 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:55.440 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:49' 2026-03-23T19:50:55.441 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:55.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:55.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:55.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:50:55.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:55.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:50:55.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:50:55.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:50:55.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:55.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:55.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:55.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:50:55.597 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:55.602 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:55.745 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:55.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:50:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: image resync while replayer is stopped' 2026-03-23T19:50:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:50:55.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:50:55.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:50:55 TEST: image resync while replayer is stopped 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:50:55 TEST: image resync while replayer is stopped 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/test 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:55.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/test 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:50:55.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:55.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:55.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:55.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:55.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:55.832 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/test 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:55.884 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:50:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:55.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:56.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:56.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:56.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:56.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:56.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:56.889 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:56.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:50:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:50:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:50:56.960 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror test image_id 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=image_id 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:56.962 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'image_id='\''176d65776e8f'\''' 2026-03-23T19:50:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ image_id=176d65776e8f 2026-03-23T19:50:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$image_id"' 2026-03-23T19:50:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 176d65776e8f 2026-03-23T19:50:57.002 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-03-23T19:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:50:57.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:57.046 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:57.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:57.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:57.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-03-23T19:50:57.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 176d65776e8f 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d65776e8f 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d65776e8f 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:57.132 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176d65776e8f 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d65776e8f 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:57.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:57.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image test: (2) No such file or directory 2026-03-23T19:50:57.268 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id= 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_instance=cluster1 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance=cluster1 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loop_instance 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:57.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:57.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:57.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:57.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:57.288 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test present 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:57.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:57.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:57.350 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:57.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=176d51c3cec2 2026-03-23T19:50:57.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:50:57.393 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present 176d51c3cec2 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:50:57.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176d51c3cec2 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d51c3cec2 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 176d51c3cec2 -o 176d51c3cec2 = 176d51c3cec2 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:57.533 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:58.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:50:58.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:58.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:50:58.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:50:58.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:50:58.538 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:50:58.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:50:58.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:58.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:58.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:50:58.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:50:58.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:50:58.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:50:58.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:50:58.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:58.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:50:58.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:59.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: cea13f2e-b2ae-4f72-b25b-171ed35679d2 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:50:57 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:50:59.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:50:59.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:50:59.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:50:59.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:50:59.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:59.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:50:59.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:59.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:50:59.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:50:59.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:50:59.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:57' 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:50:57' 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:50:57' 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:50:59.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:50:59.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING159915997mirror.0vm08.local20.2.0-712-g70f8415btrueOK59986005mirror.1vm08.local20.2.0-712-g70f8415bfalseOK60026014mirror.2vm08.local20.2.0-712-g70f8415bfalseOK60136020mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestcea13f2e-b2ae-4f72-b25b-171ed35679d2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"}59915997mirror.0vm08.local2026-03-23 19:50:57cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:50:59.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:50:59.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:50:59.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:50:57' 2026-03-23T19:50:59.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:50:59.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:50:59.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:50:59.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:50:59.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:50:59.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:59.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:50:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:50:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:50:59.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:50:59.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:59.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:59.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:50:59.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:59.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:50:59.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:50:59.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:00.043 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:00.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:00.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: request image resync while daemon is offline' 2026-03-23T19:51:00.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:51:00.062 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:51:00.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:51:00 TEST: request image resync while daemon is offline 2026-03-23T19:51:00.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:51:00 TEST: request image resync while daemon is offline 2026-03-23T19:51:00.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:51:00.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:00.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:00.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:00.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:51:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:51:00.079 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:51:00.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=97909 2026-03-23T19:51:00.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 97909 ']' 2026-03-23T19:51:00.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 97909 2026-03-23T19:51:00.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:00.080 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:00.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:00.081 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97909 '$2 == pid {print; exit 1}' 2026-03-23T19:51:00.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 97909 1.0 0.6 745516 64468 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:00.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:00.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:01.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:01.099 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97909 '$2 == pid {print; exit 1}' 2026-03-23T19:51:01.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:51:01.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:01.110 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97909 '$2 == pid {print; exit 1}' 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:01.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:51:01.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:01.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:01.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:01.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:51:01.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:01.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:01.153 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:51:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:51:01.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:01.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:51:01.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:51:01.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:51:01.185 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:51:01.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=97944 2026-03-23T19:51:01.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 97944 ']' 2026-03-23T19:51:01.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 97944 2026-03-23T19:51:01.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:01.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:01.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:01.188 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97944 '$2 == pid {print; exit 1}' 2026-03-23T19:51:01.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 97944 0.2 0.5 671760 49848 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:01.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:01.199 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:02.201 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97944 '$2 == pid {print; exit 1}' 2026-03-23T19:51:02.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:51:02.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:02.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97944 '$2 == pid {print; exit 1}' 2026-03-23T19:51:02.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:51:02.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:51:02.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:02.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:02.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:02.221 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:51:02.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:51:02.235 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:02.237 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:51:02.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:51:02.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:02.253 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:51:02.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:51:02.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:02.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:51:02.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:51:02.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:51:02.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:02.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:51:02.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:51:02.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:51:02.289 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:51:02.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=97997 2026-03-23T19:51:02.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 97997 ']' 2026-03-23T19:51:02.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 97997 2026-03-23T19:51:02.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:02.290 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:02.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:02.292 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97997 '$2 == pid {print; exit 1}' 2026-03-23T19:51:02.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 97997 0.3 0.5 671752 49904 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:02.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:02.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:03.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:03.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97997 '$2 == pid {print; exit 1}' 2026-03-23T19:51:03.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:51:03.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:03.315 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=97997 '$2 == pid {print; exit 1}' 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:03.325 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:51:03.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:51:03.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:03.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:51:03.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:51:03.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:51:03.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:51:03.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:03.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:51:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:51:03.372 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:51:03.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:51:03.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:51:03.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:51:03.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=98080 2026-03-23T19:51:03.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 98080 ']' 2026-03-23T19:51:03.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 98080 2026-03-23T19:51:03.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:03.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:03.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:03.389 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=98080 '$2 == pid {print; exit 1}' 2026-03-23T19:51:03.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 98080 0.2 0.4 674824 47640 ? Ssl 19:49 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:03.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:03.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:04.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:04.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=98080 '$2 == pid {print; exit 1}' 2026-03-23T19:51:04.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:51:04.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:51:04.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=98080 '$2 == pid {print; exit 1}' 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:51:04.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:04.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:51:04.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:51:04.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:04.445 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:51:04.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:51:04.462 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:04.464 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:51:04.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:51:04.478 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:51:04.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror test image_id 2026-03-23T19:51:04.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:04.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:04.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:04.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=image_id 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:04.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'image_id='\''176d51c3cec2'\''' 2026-03-23T19:51:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ image_id=176d51c3cec2 2026-03-23T19:51:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$image_id"' 2026-03-23T19:51:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 176d51c3cec2 2026-03-23T19:51:04.517 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-03-23T19:51:04.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:51:04.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:51:04.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:04.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:04.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:04.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:04.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test deleted 176d51c3cec2 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=deleted 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d51c3cec2 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:51:04.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176d51c3cec2 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176d51c3cec2 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d51c3cec2 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 176d51c3cec2 -o 176d51c3cec2 = 176d51c3cec2 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:51:05.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176d51c3cec2 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:06.138 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176d51c3cec2 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d51c3cec2 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 176d51c3cec2 -o 176d51c3cec2 = 176d51c3cec2 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:51:06.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:51:08.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176d51c3cec2 2026-03-23T19:51:08.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:08.174 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:08.175 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=176d51c3cec2 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 176d51c3cec2 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 176d51c3cec2 -o 176d51c3cec2 = 176d51c3cec2 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = present 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:51:08.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:51:12.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test deleted 176d51c3cec2 2026-03-23T19:51:12.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=deleted 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=176d51c3cec2 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:12.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:12.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1863ee469ba8 2026-03-23T19:51:12.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1863ee469ba8 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 176d51c3cec2 -o 176d51c3cec2 = 1863ee469ba8 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test deleted = deleted 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_present cluster1 mirror test present 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=present 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id= 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:12.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:12.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image_id=1863ee469ba8 2026-03-23T19:51:12.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:51:12.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:51:12.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_present cluster1 mirror test present 1863ee469ba8 2026-03-23T19:51:12.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:12.386 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=present 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id=1863ee469ba8 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=deleted 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_image_id 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror test 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-03-23T19:51:12.387 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_image_id=1863ee469ba8 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 1863ee469ba8 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -z 1863ee469ba8 -o 1863ee469ba8 = 1863ee469ba8 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=present 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test present = present 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:51:12.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:12.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:12.424 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test started 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:13.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:51:13.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:51:13.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:13.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:13.428 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:51:13.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:13.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:13.442 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:13.443 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-03-23T19:51:13.500 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:51:13.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:51:13.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:51:13.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:13.502 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:14.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test: 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: cea13f2e-b2ae-4f72-b25b-171ed35679d2 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:10 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:14.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:14.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:51:14.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:51:14.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"} 2026-03-23T19:51:14.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:14.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:14.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:14.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:14.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:14.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:14.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:14.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:10' 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-03-23 19:51:10' 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:10' 2026-03-23T19:51:14.567 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:14.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:14.568 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtestcea13f2e-b2ae-4f72-b25b-171ed35679d2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295397,"replay_state":"idle"}62396243mirror.0vm08.local2026-03-23 19:51:10cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-03-23T19:51:14.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:10' 2026-03-23T19:51:14.616 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-03-23T19:51:14.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:51:14.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:14.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test.export 2026-03-23T19:51:14.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test.export 2026-03-23T19:51:14.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:51:14.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:51:14.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test.export 2026-03-23T19:51:14.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test.export 2026-03-23T19:51:14.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:51:14.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:14.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:14.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export 2026-03-23T19:51:14.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:51:14.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:14.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-03-23T19:51:14.895 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:14.943 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test.export 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:14.963 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test 2026-03-23T19:51:14.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-03-23T19:51:15.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-03-23T19:51:16.573 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:51:16.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:16.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:51:16.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: split-brain' 2026-03-23T19:51:16.577 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:51:16.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:51:16.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:51:16 TEST: split-brain 2026-03-23T19:51:16.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:51:16 TEST: split-brain 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ image=split-brain 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror split-brain snapshot 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:16.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror split-brain 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:16.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/split-brain 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror split-brain snapshot 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:16.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:16.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:51:16.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/split-brain snapshot 2026-03-23T19:51:17.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:51:17.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/split-brain 2026-03-23T19:51:17.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'split-brain': 2026-03-23T19:51:17.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 128 MiB in 32 objects 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1837d21f7ff7 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1837d21f7ff7 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:51:16 2026 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:51:16 2026 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:51:16 2026 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:17.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:17.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:18.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:18.566 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:18.598 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: mirroring not enabled on the image 2026-03-23T19:51:18.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status= 2026-03-23T19:51:18.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:18.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:51:18.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:18.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:20.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:20.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"} 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:18 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:20.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:20.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:51:20.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:51:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"} 2026-03-23T19:51:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:20.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:20.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:20.660 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:20.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:20.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:20.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:18' 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-03-23 19:51:18' 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:18' 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:20.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:20.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKsplit-brain7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"}62396243mirror.0vm08.local2026-03-23 19:51:18cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:51:20.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:20.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-03-23T19:51:20.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:18' 2026-03-23T19:51:20.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-03-23T19:51:20.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:51:20.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:20.713 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror split-brain --force 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:20.714 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:20.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force=--force 2026-03-23T19:51:20.715 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/split-brain --force 2026-03-23T19:51:25.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror split-brain 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror split-brain stopped 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:25.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror split-brain stopped 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:51:26.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/split-brain 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:51:26.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/split-brain 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:26.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:51:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:26.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/split-brain 2026-03-23T19:51:26.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:51:26.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:51:26.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:51:26.753 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:51:26.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:51:26.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:26.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+stopped 2026-03-23T19:51:26.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:26.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+stopped '' '' 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:25 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 208 .mirror.primary.7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4.af06e49b-f3cf-4e78-9317-c501784b92c2 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:51:27.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:51:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:51:27.800 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:51:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:51:27.801 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:27.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:51:27.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:27.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:51:27.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:27.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:27.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:27.804 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:25' 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+stopped '2026-03-23 19:51:25' 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:25' 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:27.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKsplit-brain7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4up+stoppedlocal image is primary62396243mirror.0vm08.local2026-03-23 19:51:25cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:51:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:27.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-03-23T19:51:27.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:25' 2026-03-23T19:51:27.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-03-23T19:51:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:51:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:51:27.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster1 mirror split-brain 10 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=10 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:51:27.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 bench mirror/split-brain --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-03-23T19:51:27.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-03-23T19:51:27.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:51:27.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 10 ops/sec: 1428.57 bytes/sec: 5.6 MiB/s 2026-03-23T19:51:27.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster1 mirror split-brain 2026-03-23T19:51:27.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:27.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:27.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:27.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/split-brain 2026-03-23T19:51:28.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:51:28.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+error split-brain 2026-03-23T19:51:28.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:28.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:28.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:28.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+error 2026-03-23T19:51:28.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern=split-brain 2026-03-23T19:51:28.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:28.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:28.680 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+error 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern=split-brain 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:29.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:29.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:25 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+error' 2026-03-23T19:51:29.726 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:51:29.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:29.727 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+error 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern=split-brain 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:31.728 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:31.729 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+error 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: split-brain 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:30 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+error' 2026-03-23T19:51:31.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+error 2026-03-23T19:51:31.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*split-brain' 2026-03-23T19:51:31.775 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: split-brain 2026-03-23T19:51:31.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:31.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+error 2026-03-23T19:51:31.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:31.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+error 2026-03-23T19:51:31.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:31.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:31.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:31.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:30' 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+error '2026-03-23 19:51:30' 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+error 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:30' 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:31.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:31.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKsplit-brain7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4up+errorsplit-brain62396243mirror.0vm08.local2026-03-23 19:51:30cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:51:31.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:31.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-03-23T19:51:31.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:30' 2026-03-23T19:51:31.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-03-23T19:51:31.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+error 2026-03-23T19:51:31.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+error 2026-03-23T19:51:31.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+error 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+error 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ request_resync_image cluster1 mirror split-brain image_id 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image_id_var_name=image_id 2026-03-23T19:51:31.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_image_id cluster1 mirror split-brain 2026-03-23T19:51:31.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:31.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:31.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=split-brain 2026-03-23T19:51:31.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster=cluster1 info mirror/split-brain 2026-03-23T19:51:31.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-03-23T19:51:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'image_id='\''18632425d40b'\''' 2026-03-23T19:51:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ image_id=18632425d40b 2026-03-23T19:51:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval 'test -n "$image_id"' 2026-03-23T19:51:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -n 18632425d40b 2026-03-23T19:51:31.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/split-brain 2026-03-23T19:51:31.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Flagged image for resync from primary 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:31.908 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:32.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:32.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+error 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: split-brain 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:30 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:34.951 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:34.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+error 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: split-brain 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:30 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:34.992 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:34.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:38.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:39.023 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-03-23T19:51:39.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status= 2026-03-23T19:51:39.027 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:39.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 1 2026-03-23T19:51:39.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:39.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 8 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:47.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:47.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:47.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:47.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='split-brain: 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"} 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:40 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: down+unknown 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: status not found 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: ' 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:51:47.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"} 2026-03-23T19:51:47.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:47.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:47.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:47.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:47.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:47.282 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:47.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:47.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:40' 2026-03-23T19:51:47.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-03-23 19:51:40' 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:40' 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:47.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKsplit-brain7db1a41c-aef2-4aa2-80d4-1a39b8ed3ac4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295477,"replay_state":"idle"}62396243mirror.0vm08.local2026-03-23 19:51:40cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02down+unknownstatus not found' 2026-03-23T19:51:47.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:47.333 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-03-23T19:51:47.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:40' 2026-03-23T19:51:47.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-03-23T19:51:47.336 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:51:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:51:47.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror split-brain 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:51:47.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror split-brain 2026-03-23T19:51:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=split-brain 2026-03-23T19:51:47.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/split-brain 2026-03-23T19:51:47.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/split-brain 2026-03-23T19:51:47.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-03-23T19:51:47.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:47.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-03-23T19:51:47.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:51:47.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:51:47.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:51:47 TEST: check if removed 'images'\''' OMAP are removed 2026-03-23T19:51:47.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:51:47 TEST: check if removed images' OMAP are removed 2026-03-23T19:51:47.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster2 2026-03-23T19:51:47.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:47.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:51:47.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:47.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster2:0 ']' 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:47.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster2:1 ']' 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:47.861 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster2:2 ']' 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:47.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:47.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:47.929 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster2:3 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:3 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster2:3 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster2 2026-03-23T19:51:47.930 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:51:47.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster2:3 ']' 2026-03-23T19:51:47.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster2 2026-03-23T19:51:47.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:47.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:51:47.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:47.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:47.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=status_global 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:47.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:47.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-03-23T19:51:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:47.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=status_global 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c status_global 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:47.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:51:48.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:48.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:48.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:48.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_ 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:48.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_ 2026-03-23T19:51:48.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-03-23T19:51:48.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:48.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:48.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:48.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:51:48.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_ 2026-03-23T19:51:48.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:48.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirror_leader 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_map 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:48.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:48.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:48.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:51:48.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:48.121 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:51:48.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 0 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=1 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 -eq 0 ']' 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:48.176 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:51:49.178 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:51:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:51:49.179 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 1 = 0 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=1 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 -eq 0 ']' 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:49.210 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:51.211 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:51:51.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:51:51.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:51.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:51.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:51.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:51.242 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=status_global 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:51.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=status_global 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c status_global 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirroring 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_ 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:51.274 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:51.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:51.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-03-23T19:51:51.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_ 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_ 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirroring 2026-03-23T19:51:51.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-03-23T19:51:51.311 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local obj_name=rbd_mirror_leader 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local filter=image_map 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:51:51.312 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local filter=image_map 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -c image_map 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local obj_name=rbd_mirror_leader 2026-03-23T19:51:51.313 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 0 = 0 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ error_code=0 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set -e 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: no blocklists' 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:51:51.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:51:51.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:51:51 TEST: no blocklists 2026-03-23T19:51:51.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:51:51 TEST: no blocklists 2026-03-23T19:51:51.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS='--id admin' 2026-03-23T19:51:51.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 osd blocklist ls 2026-03-23T19:51:51.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -q 'listed 0 entries' 2026-03-23T19:51:51.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS='--id admin' 2026-03-23T19:51:51.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 osd blocklist ls 2026-03-23T19:51:51.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -q 'listed 0 entries' 2026-03-23T19:51:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:51:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:51:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: test partially synced demote snapshot sync after daemon restart' 2026-03-23T19:51:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:51:51.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:51:51.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:51:51 TEST: test partially synced demote snapshot sync after daemon restart 2026-03-23T19:51:51.969 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:51:51 TEST: test partially synced demote snapshot sync after daemon restart 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image=test_demote_image 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 10G 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_demote_image 10G 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:51:51.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 10G ']' 2026-03-23T19:51:51.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=10G 2026-03-23T19:51:51.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:51:51.971 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock mirror/test_demote_image 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:51:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-03-23T19:51:52.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:51:52.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_demote_image': 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 10 GiB in 2560 objects 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 187ac92f3ed5 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.187ac92f3ed5 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:51:52 2026 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:51:52 2026 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:51:52 2026 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:51:52.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:51:52.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:51:52.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:51:52.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:51:52.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:51:52.940 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1960.78 bytes/sec: 7.7 MiB/s 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:52.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:51:53.953 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:51:53.954 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:51:53.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:53.956 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:53.957 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:51:53.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:53.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:53.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:53.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:51:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:51:54.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:51:54.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:51:54.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:51:54.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:51:54.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:51:54.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:51:54.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:51:54.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:51:54.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:51:54.108 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:51:54.109 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:51:54.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:51:54.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:51:54.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:51:54.127 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:51:54.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:51:54.184 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:54.186 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:51:54.187 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:51:54.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:51:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:51:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:51:54.203 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-03-23T19:51:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:51:54.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:51:54.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:54.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:51:55.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:51:55.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:55.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:51:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:51:55.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:55.338 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:51:55.370 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:55.371 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:51:56.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:51:56.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:51:56.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:51:56.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:51:56.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:56.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:56.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:51:56.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:51:56.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:51:56.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:51:56.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:51:56.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:56.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:56.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-03-23T19:51:56.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 200 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-03-23T19:51:56.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:51:56.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:51:56.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-03-23T19:51:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-03-23T19:51:56.872 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:56.873 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:51:56.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:51:56.874 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:51:56.919 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:51:56.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=200 2026-03-23T19:51:56.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:51:56.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:51:56.921 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:51:57.157 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=199 2026-03-23T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 200 = 199 2026-03-23T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:51:57.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:51:57.618 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:51:57.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:51:57.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 200 = '' 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:51:57.623 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:51:58.425 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:51:58.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:51:58.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=200 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 200 = 200 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:58.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:59.476 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295512,"replay_state":"idle"} 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:55 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:58' 2026-03-23T19:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:51:59.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:51:59.528 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:51:59.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295512,"replay_state":"idle"} 2026-03-23T19:51:59.529 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:51:59.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:59.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:51:59.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:59.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:51:59.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:51:59.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:51:59.532 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:55' 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-03-23 19:51:55' 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:55' 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:51:59.534 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:51:59.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK162396243mirror.0vm08.local20.2.0-712-g70f8415bfalseOK62466249mirror.1vm08.local20.2.0-712-g70f8415btrueOK62536256mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62606263mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295512,"replay_state":"idle"}62396243mirror.0vm08.local2026-03-23 19:51:55cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:51:58' 2026-03-23T19:51:59.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:51:59.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:51:59.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:55' 2026-03-23T19:51:59.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:51:59.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:51:59.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:51:59.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:51:59.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:51:59.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:59.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:51:59.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:52:00.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:58 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295512,"replay_state":"idle"} 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:51:55 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 199 .mirror.primary.8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd.8066a642-2723-4a5d-b287-ad3964331842 (peer_uuids:[]) 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 200 .mirror.primary.8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd.995461c3-1bfe-436e-9cda-1fbc26e37ef5 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:52:00.695 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:52:00.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:52:00.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:52:00.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:52:00.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:52:00.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:52:00.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:52:00.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:52:00.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:52:00.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:52:00.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:52:00.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:52:00.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:58' 2026-03-23T19:52:00.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-03-23 19:51:58' 2026-03-23T19:52:00.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:51:58' 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:52:00.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:52:00.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK162156217mirror.0vm08.local20.2.0-712-g70f8415btrueOK62216227mirror.1vm08.local20.2.0-712-g70f8415bfalseOK62286234mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62366239mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+stoppedlocal image is primary62156217mirror.0vm08.local2026-03-23 19:51:58cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295512,"replay_state":"idle"}2026-03-23 19:51:55' 2026-03-23T19:52:00.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:52:00.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:52:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:51:58' 2026-03-23T19:52:00.749 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:52:00.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:52:00.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:52:00.752 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_demote_image 2560 4194304 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=2560 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=4194304 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 4194304 2026-03-23T19:52:00.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-03-23T19:52:00.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-03-23T19:52:01.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:52:01.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 1 56 60.3174 241 MiB/s 2026-03-23T19:52:02.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 2 123 63.5245 254 MiB/s 2026-03-23T19:52:03.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 3 189 64.4941 258 MiB/s 2026-03-23T19:52:04.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 4 249 63.4356 254 MiB/s 2026-03-23T19:52:05.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 5 308 62.5632 250 MiB/s 2026-03-23T19:52:06.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 6 369 62.5124 250 MiB/s 2026-03-23T19:52:07.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 7 432 60.3161 241 MiB/s 2026-03-23T19:52:08.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 8 450 51.8577 207 MiB/s 2026-03-23T19:52:09.811 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 9 516 53.4641 214 MiB/s 2026-03-23T19:52:10.813 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 10 584 55.222 221 MiB/s 2026-03-23T19:52:11.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 11 619 50.1504 201 MiB/s 2026-03-23T19:52:12.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 12 683 51.6779 207 MiB/s 2026-03-23T19:52:13.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 13 714 51.9787 208 MiB/s 2026-03-23T19:52:14.853 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 14 773 50.9718 204 MiB/s 2026-03-23T19:52:15.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 15 840 51.1079 204 MiB/s 2026-03-23T19:52:16.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 16 913 58.4608 234 MiB/s 2026-03-23T19:52:17.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 17 977 58.554 234 MiB/s 2026-03-23T19:52:18.842 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 18 1047 67.8621 271 MiB/s 2026-03-23T19:52:19.817 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 19 1104 66.6934 267 MiB/s 2026-03-23T19:52:20.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 20 1167 65.3346 261 MiB/s 2026-03-23T19:52:21.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 21 1219 61.5075 246 MiB/s 2026-03-23T19:52:22.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 22 1269 57.9709 232 MiB/s 2026-03-23T19:52:23.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 23 1332 57.3671 229 MiB/s 2026-03-23T19:52:24.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 24 1404 59.9639 240 MiB/s 2026-03-23T19:52:25.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 25 1459 58.3882 234 MiB/s 2026-03-23T19:52:26.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 26 1524 60.9512 244 MiB/s 2026-03-23T19:52:27.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 27 1591 64.9324 260 MiB/s 2026-03-23T19:52:28.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 28 1621 57.0582 228 MiB/s 2026-03-23T19:52:29.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 29 1679 54.945 220 MiB/s 2026-03-23T19:52:30.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 30 1716 51.1239 204 MiB/s 2026-03-23T19:52:31.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 31 1772 49.6099 198 MiB/s 2026-03-23T19:52:32.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 32 1832 47.2363 189 MiB/s 2026-03-23T19:52:33.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 33 1873 50.9605 204 MiB/s 2026-03-23T19:52:34.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 34 1937 51.693 207 MiB/s 2026-03-23T19:52:35.857 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 35 1977 52.1895 209 MiB/s 2026-03-23T19:52:36.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 36 2003 45.1259 181 MiB/s 2026-03-23T19:52:37.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 37 2065 47.6677 191 MiB/s 2026-03-23T19:52:38.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 38 2103 45.899 184 MiB/s 2026-03-23T19:52:39.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 39 2164 45.3636 181 MiB/s 2026-03-23T19:52:40.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 40 2228 50.5946 202 MiB/s 2026-03-23T19:52:41.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 41 2294 59.5213 238 MiB/s 2026-03-23T19:52:42.808 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 42 2364 59.8838 240 MiB/s 2026-03-23T19:52:43.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 43 2435 65.5996 262 MiB/s 2026-03-23T19:52:44.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 44 2503 67.476 270 MiB/s 2026-03-23T19:52:45.691 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 44 ops: 2560 ops/sec: 57.0371 bytes/sec: 228 MiB/s 2026-03-23T19:52:45.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ demote_image cluster2 mirror test_demote_image 2026-03-23T19:52:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:45.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-03-23T19:52:47.134 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image demoted to non-primary 2026-03-23T19:52:47.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_complete_mirror_snapshot_id cluster2 mirror test_demote_image primary_snap_id 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local -n _snap_id=primary_snap_id 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:47.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '(//snapshots/snapshot[namespace/complete='\''true'\'']/id)[last()]' 2026-03-23T19:52:47.205 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ _snap_id=201 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_non_primary_snap_present cluster1 mirror test_demote_image 201 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snap_id=201 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-03-23T19:52:47.293 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.1 2026-03-23T19:52:47.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:47.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot/namespace[primary_snap_id='\''201'\'']' 2026-03-23T19:52:47.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:52:47.480 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 -KILL 2026-03-23T19:52:47.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:47.482 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:52:47.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:52:47.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 -KILL 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:52:47.490 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:52:47.491 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:52:47.521 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:52:47.522 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:52:47.525 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=101498 2026-03-23T19:52:47.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 101498 ']' 2026-03-23T19:52:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 101498 2026-03-23T19:52:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:52:47.530 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:52:47.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.531 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101498 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:52:47.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.547 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101498 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:52:47.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:52:47.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:52:47.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:52:47.639 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 -KILL 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=101542 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 101542 ']' 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 101542 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101542 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:52:47.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.642 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101542 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:52:47.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:52:47.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:52:47.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:52:47.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:52:47.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:52:47.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:52:47.686 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:52:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:52:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:52:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:52:47.687 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:52:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:52:47.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 -KILL 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:52:47.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:52:47.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:52:47.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:52:47.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:52:47.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=101593 2026-03-23T19:52:47.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 101593 ']' 2026-03-23T19:52:47.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 101593 2026-03-23T19:52:47.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:52:47.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:52:47.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.721 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101593 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:52:47.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.732 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101593 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:52:47.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:52:47.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:52:47.762 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:52:47.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:52:47.778 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:52:47.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:52:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:52:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:52:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:52:47.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:52:47.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:52:47.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:52:47.796 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 -KILL 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:52:47.798 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:52:47.799 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:52:47.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:52:47.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=101653 2026-03-23T19:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 101653 ']' 2026-03-23T19:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 101653 2026-03-23T19:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101653 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:52:47.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:52:47.837 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=101653 '$2 == pid {print; exit 1}' 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:52:47.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:52:47.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:52:47.865 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:52:47.866 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:52:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:52:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:52:47.867 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:52:47.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:52:47.885 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:52:47.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:52:47.886 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:52:47.887 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:52:47.904 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:52:47.905 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:52:47.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_snaps_json cluster1 mirror test_demote_image 2026-03-23T19:52:47.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:52:47.906 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:52:47.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_demote_image 2026-03-23T19:52:47.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 snap ls mirror/test_demote_image --all --format json 2026-03-23T19:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ SNAPS='[{"id":212,"name":".mirror.non_primary.8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd.f2672df8-1fe8-4e26-9b59-c8293cb615aa","size":10737418240,"protected":"false","timestamp":"Mon Mar 23 19:51:57 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"42224bdf-e849-40ef-873a-b5e8f8e8ff02","primary_snap_id":200,"last_copied_object_number":2560}},{"id":213,"name":".mirror.non_primary.8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd.7a8eaa0a-e1b0-42df-8aa0-5cbfeecb8c07","size":10737418240,"protected":"false","timestamp":"Mon Mar 23 19:52:47 2026","namespace":{"type":"mirror","state":"demoted","mirror_peer_uuids":["04da4782-4604-444b-8d54-7d24cb321a84"],"complete":false,"primary_mirror_uuid":"42224bdf-e849-40ef-873a-b5e8f8e8ff02","primary_snap_id":201,"last_copied_object_number":1}}]' 2026-03-23T19:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' 2026-03-23T19:52:47.964 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:mirrordemoted04da4782-4604-444b-8d54-7d24cb321a84false42224bdf-e849-40ef-873a-b5e8f8e8ff022010true 2026-03-23T19:52:47.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:52:47.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:47.965 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.966 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:47.996 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:52:48.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:52:48.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:52:48.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:48.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-03-23T19:52:48.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:52:48.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-03-23T19:52:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:52:48.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:52:48.128 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.status 2026-03-23T19:52:48.129 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:48.130 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:48.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:52:48.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=201 2026-03-23T19:52:48.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:48.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:48.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:48.416 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:48.455 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:48.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:48.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:48.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:48.859 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:48.860 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:48.901 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:48.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:48.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:52:49.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:49.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:49.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:49.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:49.705 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:49.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:49.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:49.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:49.754 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:49.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:49.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:49.756 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:49.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:49.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:49.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:49.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1.6 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:51.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:51.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:51.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:51.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:51.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:51.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:51.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:51.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:51.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:51.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:52:53.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:53.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:53.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:53.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:53.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:53.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:53.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:53.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:53.504 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:53.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:53.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:53.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:55.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:55.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:55.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:55.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:55.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:55.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:55.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:55.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:52:59.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:52:59.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:52:59.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:52:59.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:52:59.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:52:59.607 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:52:59.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:52:59.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:52:59.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:52:59.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 4 2026-03-23T19:53:03.609 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:53:03.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:53:03.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:53:03.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:53:03.653 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 8 2026-03-23T19:53:11.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:53:11.656 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:53:11.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:53:11.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:53:11.706 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 8 2026-03-23T19:53:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:53:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:53:19.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:53:19.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:19.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:53:19.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:53:19.759 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:53:19.760 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:53:19.761 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 16 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:53:35.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:53:35.814 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:53:35.815 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:53:35.816 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 16 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:53:51.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:53:52.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:53:52.404 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:53:52.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:53:52.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:53:52.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:53:52.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:53:52.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = '' 2026-03-23T19:53:52.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:53:52.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 32 2026-03-23T19:54:03.171 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:54:03.169+0000 7f9862b48640 -1 reset not still connected to 0x55d6bff62340 2026-03-23T19:54:03.172 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:54:03.169+0000 7f9862b48640 -1 reset not still connected to 0x55d6c226de10 2026-03-23T19:54:03.172 INFO:tasks.ceph.cluster1.osd.2.vm04.stderr:2026-03-23T19:54:03.169+0000 7f9862b48640 -1 reset not still connected to 0x55d6c5ca60d0 2026-03-23T19:54:24.408 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.status 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-03-23T19:54:24.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:54:24.454 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:54:24.456 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=201 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 201 = 201 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:54:24.457 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:54:25.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-03-23T19:54:25.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:54:25.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:25.458 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:54:25.459 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:54:23 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:54:23' 2026-03-23T19:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:54:25.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:54:25.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:54:25.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:54:25.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:54:25.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:54:25.508 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:54:25.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:54:25.509 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:54:25.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:54:25.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:54:25.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:54:23' 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-03-23 19:54:23' 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:54:23' 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:54:25.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:54:25.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:54:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING162156217mirror.0vm08.local20.2.0-712-g70f8415btrueOK62216227mirror.1vm08.local20.2.0-712-g70f8415bfalseOK62286234mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62366239mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+unknownremote image is not primary62156217mirror.0vm08.local2026-03-23 19:54:23cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+unknownremote image is not primary2026-03-23 19:54:23' 2026-03-23T19:54:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:54:25.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:54:25.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:54:23' 2026-03-23T19:54:25.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:54:25.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:54:25.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:54:25.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:25.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:25.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:54:25.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:54:25.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:54:25.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:54:26.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:54:26.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:54:23 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+unknown 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: remote image is not primary 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:54:23' 2026-03-23T19:54:26.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+unknown' 2026-03-23T19:54:26.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+unknown 2026-03-23T19:54:26.611 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:54:26.612 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: remote image is not primary 2026-03-23T19:54:26.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:54:26.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:54:26.613 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:54:26.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:54:26.614 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:54:26.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:54:26.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:54:26.615 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:54:23' 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-03-23 19:54:23' 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+unknown 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:54:23' 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:54:26.617 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:54:26.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='WARNINGOKWARNING164206427mirror.0vm08.local20.2.0-712-g70f8415bfalseOK64266434mirror.1vm08.local20.2.0-712-g70f8415bfalseOK64336442mirror.2vm08.local20.2.0-712-g70f8415bfalseOK64416448mirror.3vm08.local20.2.0-712-g70f8415btrueOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+unknownremote image is not primary64206427mirror.0vm08.local2026-03-23 19:54:23cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+unknownremote image is not primary2026-03-23 19:54:23' 2026-03-23T19:54:26.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:54:26.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:54:26.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:54:23' 2026-03-23T19:54:26.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:54:26.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+unknown 2026-03-23T19:54:26.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+unknown 2026-03-23T19:54:26.671 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+unknown 2026-03-23T19:54:26.672 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+unknown 2026-03-23T19:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ compare_images cluster1 cluster2 mirror mirror test_demote_image 2026-03-23T19:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local ret=0 2026-03-23T19:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-03-23T19:54:26.674 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:54:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local rmt_export=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export 2026-03-23T19:54:26.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-03-23T19:54:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-03-23T19:54:26.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:54:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local loc_export=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:54:26.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:54:26.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export 2026-03-23T19:54:52.306 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-23T19:54:52.326 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:55:05.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-03-23T19:55:05.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cmp /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:55:18.661 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_demote_image.export /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_demote_image.export 2026-03-23T19:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_demote_image 2026-03-23T19:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force= 2026-03-23T19:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-03-23T19:55:20.213 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:20.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:21.225 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:55:21.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:55:21.226 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:55:21.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:21.227 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:55:21.229 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:55:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:21.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:55:21.373 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:55:21.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:55:21.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:55:21.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:55:21.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = stopped 2026-03-23T19:55:21.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:21.375 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 2 2026-03-23T19:55:23.376 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:23.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:55:23.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:55:23.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:55:23.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:55:23.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:55:23.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:23.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:23.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:23.394 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-03-23T19:55:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:55:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:55:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:55:23.448 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:23.450 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:24.451 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-03-23T19:55:24.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:23 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295720,"replay_state":"idle"} 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:23 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 214 .mirror.primary.8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd.10e80030-a823-499a-9052-ba5b6ae87336 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:55:24.557 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:55:24.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:55:24.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:55:24.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:55:24.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:55:24.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:55:24.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:55:24.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:55:24.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:55:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:55:24.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:23' 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-03-23 19:55:23' 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:24.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:24.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:55:23' 2026-03-23T19:55:24.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:24.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:24.564 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:55:24.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK164206427mirror.0vm08.local20.2.0-712-g70f8415bfalseOK64266434mirror.1vm08.local20.2.0-712-g70f8415bfalseOK64336442mirror.2vm08.local20.2.0-712-g70f8415bfalseOK64416448mirror.3vm08.local20.2.0-712-g70f8415btrueOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+stoppedlocal image is primary64206427mirror.0vm08.local2026-03-23 19:55:23cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295720,"replay_state":"idle"}2026-03-23 19:55:23' 2026-03-23T19:55:24.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:55:24.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:55:24.654 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:23' 2026-03-23T19:55:24.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:55:24.658 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:55:24.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:55:24.659 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:55:24.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:24.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:25.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:25.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_demote_image: 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: 8ba8ee80-0d77-4c91-98c2-a0fb2734e9bd 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295720,"replay_state":"idle"} 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:23 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:23' 2026-03-23T19:55:26.045 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:55:26.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:55:26.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:55:26.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295720,"replay_state":"idle"} 2026-03-23T19:55:26.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:55:26.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:55:26.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:55:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:55:26.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:55:26.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:55:26.056 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:55:26.057 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:23' 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-03-23 19:55:23' 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:55:23' 2026-03-23T19:55:26.058 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:26.059 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:55:26.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK162156217mirror.0vm08.local20.2.0-712-g70f8415btrueOK62216227mirror.1vm08.local20.2.0-712-g70f8415bfalseOK62286234mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62366239mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_demote_image8ba8ee80-0d77-4c91-98c2-a0fb2734e9bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295720,"replay_state":"idle"}62156217mirror.0vm08.local2026-03-23 19:55:23cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:55:23' 2026-03-23T19:55:26.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:55:26.168 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-03-23T19:55:26.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:23' 2026-03-23T19:55:26.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-03-23T19:55:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:55:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:55:26.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:55:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:55:26.172 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_demote_image 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_demote_image 2026-03-23T19:55:26.173 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-03-23T19:55:26.243 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-03-23T19:55:30.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-23T19:55:30.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:30.844 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ testlog 'TEST: force promote with a user snapshot' 2026-03-23T19:55:30.845 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ tee -a /tmp/tmp.ZXt03XlClS/rbd-mirror.test.log 2026-03-23T19:55:30.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ date '+%F %T' 2026-03-23T19:55:30.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo 2026-03-23 19:55:30 TEST: force promote with a user snapshot 2026-03-23T19:55:30.848 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:2026-03-23 19:55:30 TEST: force promote with a user snapshot 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ force_promote_image=test_force_promote_user 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote_user snapshot 10G 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n snapshot ']' 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_image cluster2 mirror test_force_promote_user 10G 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=128 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 10G ']' 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=10G 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ shift 2026-03-23T19:55:30.849 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock mirror/test_force_promote_user 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ is_pool_mirror_mode_image mirror 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' image = image ']' 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ enable_mirror cluster2 mirror test_force_promote_user snapshot 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local mode=snapshot 2026-03-23T19:55:30.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote_user snapshot 2026-03-23T19:55:31.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Mirroring enabled 2026-03-23T19:55:31.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote_user 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_force_promote_user': 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 10 GiB in 2560 objects 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 1 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 18bacfb3a85c 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.18bacfb3a85c 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:55:30 2026 2026-03-23T19:55:31.539 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:55:30 2026 2026-03-23T19:55:31.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:55:30 2026 2026-03-23T19:55:31.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:55:31.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:55:31.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:55:31.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: true 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:55:31.543 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:55:31.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:55:31.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:55:31.544 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:55:31.585 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:55:31.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:55:31.640 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1851.85 bytes/sec: 7.2 MiB/s 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:31.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:55:32.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:32.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:55:32.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:55:32.649 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:55:32.650 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:55:32.651 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:55:32.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:32.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:32.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:32.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:32.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:32.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:32.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:55:32.712 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:55:32.724 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:32.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:32.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:32.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.772 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:32.779 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:55:32.780 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:55:32.781 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:55:32.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:55:32.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:55:32.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:55:32.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:32.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:32.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:3 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:32.840 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:55:32.841 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:55:32.855 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:55:32.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:55:32.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:55:32.869 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:32.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:32.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:32.907 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:32.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:32.909 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:55:33.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:55:33.910 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:55:33.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:55:33.948 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:33.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:33.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:55:33.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:34.019 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:55:34.028 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:55:34.029 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:55:34.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:34.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:34.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:55:34.035 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:34.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:55:34.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:55:34.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:55:34.090 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote_user 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user started 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=started 2026-03-23T19:55:34.091 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:55:34.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:34.092 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user started 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=started 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:55:35.093 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:55:35.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:35.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:35.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:55:35.095 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:55:35.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:55:35.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:55:35.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:55:35.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:55:35.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:55:35.097 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:55:35.131 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:55:35.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:55:35.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:55:35.133 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:55:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:55:35.220 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:55:35.223 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:55:35.224 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:55:35.245 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:55:35.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:55:35.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:55:35.246 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:55:35.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:55:35.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Replaying"' 2026-03-23T19:55:35.341 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:55:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Replaying"' 2026-03-23T19:55:35.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: "state": "Replaying" 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ current_state=started 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test started = started 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_force_promote_user 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:35.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-03-23T19:55:35.970 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 204 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_force_promote_user 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_cluster=cluster1 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_pool=mirror 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local remote_pool=mirror 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:35.977 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster2-mirror-test_force_promote_user.status 2026-03-23T19:55:35.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster2-mirror-test_force_promote_user.status 2026-03-23T19:55:35.978 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:55:35.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote_user.status 2026-03-23T19:55:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ mkfname cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:35.980 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -e 's|[/ ]|_|g' 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local local_status_log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote_user /tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote_user.status 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster2-mirror-test_force_promote_user.status 2026-03-23T19:55:35.982 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote_user --format xml 2026-03-23T19:55:35.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:55:36.094 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-03-23T19:55:36.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snapshot_id=204 2026-03-23T19:55:36.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:55:36.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:55:36.096 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.2 2026-03-23T19:55:36.297 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-03-23T19:55:36.298 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:55:36.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:55:36.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=203 2026-03-23T19:55:36.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 204 = 203 2026-03-23T19:55:36.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:55:36.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.4 2026-03-23T19:55:36.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:36.746 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:36.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:36.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:36.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:36.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-03-23T19:55:36.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:55:36.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ true 2026-03-23T19:55:36.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:-:1.1: Document is empty 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:^ 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id= 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 204 = '' 2026-03-23T19:55:36.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-03-23T19:55:36.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0.8 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local log=/tmp/tmp.ZXt03XlClS/cluster1-mirror-test_force_promote_user.status 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-03-23T19:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-03-23T19:55:37.673 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local primary_snapshot_id=204 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test 204 = 204 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote_user 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = journal ']' 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:37.675 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying '' '' 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:38.676 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote_user: 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295731,"replay_state":"idle"} 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:33 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:38' 2026-03-23T19:55:39.047 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+replaying' 2026-03-23T19:55:39.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+replaying 2026-03-23T19:55:39.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:55:39.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295731,"replay_state":"idle"} 2026-03-23T19:55:39.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:55:39.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:55:39.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:55:39.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:55:39.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:55:39.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:55:39.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:55:39.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:33' 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+replaying '2026-03-23 19:55:33' 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+replaying 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:55:33' 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:39.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:55:39.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK164206427mirror.0vm08.local20.2.0-712-g70f8415bfalseOK64266434mirror.1vm08.local20.2.0-712-g70f8415bfalseOK64336442mirror.2vm08.local20.2.0-712-g70f8415bfalseOK64416448mirror.3vm08.local20.2.0-712-g70f8415btrueOKtest_force_promote_userd1fc621b-fd26-4399-882d-c490a7606324up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295731,"replay_state":"idle"}64266434mirror.1vm08.local2026-03-23 19:55:33cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:55:38' 2026-03-23T19:55:39.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:55:39.098 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-03-23T19:55:39.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:33' 2026-03-23T19:55:39.100 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-03-23T19:55:39.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+replaying 2026-03-23T19:55:39.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+replaying 2026-03-23T19:55:39.102 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+replaying 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+replaying 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:55:39.103 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:55:40.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-03-23T19:55:40.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:40.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:40.104 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:40.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote_user: 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:55:40.146 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:38 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+replaying 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295731,"replay_state":"idle"} 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:55:33 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 203 .mirror.primary.d1fc621b-fd26-4399-882d-c490a7606324.4dd846bc-da84-415e-9225-b621a0b19f04 (peer_uuids:[]) 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 204 .mirror.primary.d1fc621b-fd26-4399-882d-c490a7606324.dfd98ad9-1d9b-4184-ac6f-39884e55d8d5 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:55:40.147 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:55:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:55:40.148 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:55:40.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:55:40.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:55:40.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:55:40.149 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:55:40.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:55:40.150 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:55:40.151 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:38' 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-03-23 19:55:38' 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:55:38' 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:55:40.152 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:55:40.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK162156217mirror.0vm08.local20.2.0-712-g70f8415btrueOK62216227mirror.1vm08.local20.2.0-712-g70f8415bfalseOK62286234mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62366239mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_force_promote_userd1fc621b-fd26-4399-882d-c490a7606324up+stoppedlocal image is primary62216227mirror.1vm08.local2026-03-23 19:55:38cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1774295731,"replay_state":"idle"}2026-03-23 19:55:33' 2026-03-23T19:55:40.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:55:40.202 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-03-23T19:55:40.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:55:38' 2026-03-23T19:55:40.204 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-03-23T19:55:40.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:55:40.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:55:40.206 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:55:40.207 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:55:40.248 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:55:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:55:40.300 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 1923.07 bytes/sec: 7.5 MiB/s 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ create_snapshot cluster2 mirror test_force_promote_user snap1 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap=snap1 2026-03-23T19:55:40.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 snap create mirror/test_force_promote_user@snap1 2026-03-23T19:55:41.200 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote_user 2560 4194304 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=2560 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size=4194304 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n 4194304 2026-03-23T19:55:41.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-03-23T19:55:41.249 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-03-23T19:55:42.250 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:55:42.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 1 76 77.077 308 MiB/s 2026-03-23T19:55:43.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 2 151 75.2847 301 MiB/s 2026-03-23T19:55:44.252 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 3 216 72.3332 289 MiB/s 2026-03-23T19:55:45.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 4 292 72.7227 291 MiB/s 2026-03-23T19:55:46.349 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 5 366 71.9748 288 MiB/s 2026-03-23T19:55:47.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 6 429 70.2626 281 MiB/s 2026-03-23T19:55:48.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 7 511 72.1876 289 MiB/s 2026-03-23T19:55:49.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 8 598 75.6285 303 MiB/s 2026-03-23T19:55:50.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 9 675 77.0003 308 MiB/s 2026-03-23T19:55:51.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 10 768 81.7737 327 MiB/s 2026-03-23T19:55:52.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 11 862 86.7908 347 MiB/s 2026-03-23T19:55:53.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 12 950 87.9054 352 MiB/s 2026-03-23T19:55:54.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 13 1020 85.132 341 MiB/s 2026-03-23T19:55:55.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 14 1071 79.2474 317 MiB/s 2026-03-23T19:55:56.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 15 1133 71.0116 284 MiB/s 2026-03-23T19:55:57.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 16 1209 69.4138 278 MiB/s 2026-03-23T19:55:58.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 17 1306 71.0294 284 MiB/s 2026-03-23T19:55:59.264 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 18 1384 72.7272 291 MiB/s 2026-03-23T19:56:00.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 19 1466 78.6225 314 MiB/s 2026-03-23T19:56:01.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 20 1530 81.7881 327 MiB/s 2026-03-23T19:56:02.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 21 1624 82.6198 330 MiB/s 2026-03-23T19:56:03.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 22 1705 79.8478 319 MiB/s 2026-03-23T19:56:04.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 23 1785 80.2641 321 MiB/s 2026-03-23T19:56:05.254 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 24 1851 77.3091 309 MiB/s 2026-03-23T19:56:06.251 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 25 1933 80.7452 323 MiB/s 2026-03-23T19:56:07.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 26 2005 76.5367 306 MiB/s 2026-03-23T19:56:08.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 27 2072 73.3559 293 MiB/s 2026-03-23T19:56:09.275 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 28 2136 69.9899 280 MiB/s 2026-03-23T19:56:10.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 29 2210 71.7712 287 MiB/s 2026-03-23T19:56:11.327 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 30 2288 69.9369 280 MiB/s 2026-03-23T19:56:12.321 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 31 2296 57.5325 230 MiB/s 2026-03-23T19:56:13.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 32 2370 59.6715 239 MiB/s 2026-03-23T19:56:14.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 33 2446 62.2239 249 MiB/s 2026-03-23T19:56:15.276 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: 34 2520 61.7529 247 MiB/s 2026-03-23T19:56:15.850 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 34 ops: 2560 ops/sec: 73.9883 bytes/sec: 296 MiB/s 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:15.858 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-03-23T19:56:17.273 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Snapshot ID: 206 2026-03-23T19:56:17.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_snap_present cluster1 mirror test_force_promote_user snap1 2026-03-23T19:56:17.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:17.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:17.279 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:17.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local snap_name=snap1 2026-03-23T19:56:17.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:56:17.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-03-23T19:56:17.280 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:56:18.281 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 info mirror/test_force_promote_user@snap1 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:rbd image 'test_force_promote_user': 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: size 10 GiB in 2560 objects 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: order 22 (4 MiB objects) 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: snapshot_count: 3 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: id: 1922720ff369 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: block_name_prefix: rbd_data.1922720ff369 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: format: 2 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: features: layering, exclusive-lock, non-primary 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: op_features: 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: flags: 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: create_timestamp: Mon Mar 23 19:55:32 2026 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: access_timestamp: Mon Mar 23 19:55:32 2026 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: modify_timestamp: Mon Mar 23 19:55:32 2026 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: protected: False 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring state: enabled 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring mode: snapshot 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring global id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:56:18.335 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: mirroring primary: false 2026-03-23T19:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:56:19.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 -KILL 2026-03-23T19:56:19.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:19.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:56:19.342 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:56:19.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:19.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 -KILL 2026-03-23T19:56:19.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:56:19.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:56:19.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:19.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:56:19.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:56:19.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:56:19.374 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:56:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=103964 2026-03-23T19:56:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 103964 ']' 2026-03-23T19:56:19.377 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 103964 2026-03-23T19:56:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:56:19.378 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:56:19.379 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.380 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=103964 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:56:19.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.412 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=103964 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:19.426 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:19.427 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:56:19.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:19.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:19.446 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:56:19.471 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:19.472 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:19.474 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:56:19.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:56:19.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 -KILL 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:56:19.498 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:56:19.515 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:56:19.516 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:56:19.519 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=103992 2026-03-23T19:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 103992 ']' 2026-03-23T19:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 103992 2026-03-23T19:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=103992 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:56:19.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.540 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=103992 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:19.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:56:19.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:19.569 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:19.571 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:56:19.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:19.587 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:19.589 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:56:19.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:56:19.603 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 -KILL 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:19.604 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:56:19.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:56:19.619 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:56:19.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=104052 2026-03-23T19:56:19.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 104052 ']' 2026-03-23T19:56:19.620 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 104052 2026-03-23T19:56:19.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:56:19.621 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:56:19.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.622 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=104052 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.633 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:56:19.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.634 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=104052 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:19.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:19.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:56:19.662 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:19.663 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:19.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:19.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:56:19.681 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:19.682 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:19.683 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:19.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:56:19.699 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:56:19.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:56:19.700 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 -KILL 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig=-KILL 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:19.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:56:19.702 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:56:19.716 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:56:19.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:56:19.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=104125 2026-03-23T19:56:19.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 104125 ']' 2026-03-23T19:56:19.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill -KILL 104125 2026-03-23T19:56:19.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:56:19.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:56:19.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=104125 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:56:19.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:56:19.730 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=104125 '$2 == pid {print; exit 1}' 2026-03-23T19:56:19.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:56:19.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:56:19.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:19.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.740 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:19.741 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:56:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:19.755 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:19.757 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:56:19.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:19.774 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:19.776 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:19.777 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:56:19.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:56:19.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:56:19.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' snapshot = snapshot ']' 2026-03-23T19:56:19.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ get_snaps_json cluster1 mirror test_force_promote_user 2026-03-23T19:56:19.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:19.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local pool=mirror 2026-03-23T19:56:19.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local image=test_force_promote_user 2026-03-23T19:56:19.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 snap ls mirror/test_force_promote_user --all --format json 2026-03-23T19:56:19.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ SNAPS='[{"id":216,"name":".mirror.non_primary.d1fc621b-fd26-4399-882d-c490a7606324.15745d1a-f4d1-4f66-8f6c-8f9da9fe6876","size":10737418240,"protected":"false","timestamp":"Mon Mar 23 19:55:36 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"42224bdf-e849-40ef-873a-b5e8f8e8ff02","primary_snap_id":204,"last_copied_object_number":2560}},{"id":217,"name":"snap1","size":10737418240,"protected":"false","timestamp":"Mon Mar 23 19:56:16 2026","namespace":{"type":"user"}},{"id":218,"name":".mirror.non_primary.d1fc621b-fd26-4399-882d-c490a7606324.e2f82c9b-ba6a-4ef7-a40c-43f78698c435","size":10737418240,"protected":"false","timestamp":"Mon Mar 23 19:56:17 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":false,"primary_mirror_uuid":"42224bdf-e849-40ef-873a-b5e8f8e8ff02","primary_snap_id":206,"last_copied_object_number":105}}]' 2026-03-23T19:56:19.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "non-primary" and .[-1].namespace["complete"] == false' 2026-03-23T19:56:19.838 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:true 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ promote_image cluster1 mirror test_force_promote_user --force 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local force=--force 2026-03-23T19:56:19.839 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote_user --force 2026-03-23T19:56:54.527 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:Image promoted to primary 2026-03-23T19:56:54.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirrors cluster1 2026-03-23T19:56:54.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:54.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:54.537 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:54.565 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:56:54.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:54.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ start_mirror cluster1:3 2026-03-23T19:56:54.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:56:54.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local val=cluster1:3 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster_var_name=cluster 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local instance_var_name=instance 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cluster=cluster1 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ instance=3 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval cluster=cluster1 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ eval instance=3 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:56:54.629 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-03-23T19:56:54.664 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote_user 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:56:54.665 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster1 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster1 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:56:55.666 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:56:55.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:56:55.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:55.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:56:55.669 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:56:55.684 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:55.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:55.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:55.685 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:55.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:55.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:55.750 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:56:55.751 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:56:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:55.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.823 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:55.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:55.824 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:55.825 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:56:55.826 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:56:55.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:55.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:55.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:55.847 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:55.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:55.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.902 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:56:55.903 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:56:55.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:55.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:55.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:55.927 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:55.981 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:56:55.983 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 1 2026-03-23T19:56:56.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:56:56.984 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:56:56.985 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:56:56.986 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:56:56.987 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:56:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:57.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:57.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:57.052 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:56:57.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:56:57.069 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:57.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:57.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:57.070 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:57.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:57.125 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:56:57.126 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:56:57.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:57.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:57.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:57.142 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:57.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:57.190 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:57.192 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:56:57.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:56:57.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:57.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:57.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:57.209 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:57.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:57.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:57.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:57.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:56:57.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 2 2026-03-23T19:56:59.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:56:59.263 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:59.265 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:56:59.266 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:56:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:56:59.283 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:56:59.339 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:56:59.340 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:56:59.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:59.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:59.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:56:59.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:59.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:59.409 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:56:59.413 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:56:59.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:56:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:56:59.433 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:59.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:59.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:56:59.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:56:59.489 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:56:59.505 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:59.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:59.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:56:59.506 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:56:59.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:56:59.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:56:59.558 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:56:59.559 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:56:59.560 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 4 2026-03-23T19:57:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:57:03.561 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:03.562 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:03.563 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:03.578 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:03.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:03.636 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:03.637 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:03.638 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:03.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:03.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:03.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:03.652 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:03.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:03.701 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:03.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:03.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:03.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:03.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:03.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:03.704 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:03.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:03.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:03.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:03.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.763 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:03.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:03.834 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:57:03.835 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:03.836 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 8 2026-03-23T19:57:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:57:11.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:11.830 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:11.831 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:11.899 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:11.900 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:11.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:11.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:11.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:11.917 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:11.973 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:11.975 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:11.976 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:11.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:11.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:11.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:11.993 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:12.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:12.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:12.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:12.049 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:12.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:12.051 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:12.065 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:12.115 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:12.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:12.116 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 8 2026-03-23T19:57:20.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:57:20.118 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:20.119 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:20.120 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:20.136 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:20.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:20.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:20.137 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:20.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:20.191 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:20.193 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:57:20.194 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:20.195 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:20.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:20.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:20.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:20.212 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:20.269 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:20.271 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:20.286 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:20.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:20.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:20.287 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:20.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:20.343 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:20.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:20.344 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:20.345 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:20.346 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:20.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:20.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:20.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:20.362 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:20.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:20.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:20.414 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:20.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:20.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 8 2026-03-23T19:57:28.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:57:28.417 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:28.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:28.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:28.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:28.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:28.436 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:28.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:28.484 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.485 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:28.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:28.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:28.487 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:28.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:28.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:28.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:28.501 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:28.556 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:28.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:28.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:28.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:28.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:28.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:28.628 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:57:28.630 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:28.631 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:28.647 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 8 2026-03-23T19:57:36.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-03-23T19:57:36.709 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:36.710 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster1:0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster1 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster1 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:36.711 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:36.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:36.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:36.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:36.725 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:36.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:57:36.789 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:57:36.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:57:36.790 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state=stopped 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local s 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:36.792 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:57:37.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-03-23T19:57:37.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:37.793 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local test_state=stopped 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status_result 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local current_state=stopped 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_instance=cluster2 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance=cluster2 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local loop_instance 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sleep 0 2026-03-23T19:57:37.794 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ grep -i state 2026-03-23T19:57:37.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-03-23T19:57:37.795 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ seq 0 3 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:0 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:37.797 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:57:37.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:57:37.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:37.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:37.802 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:37.803 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:57:37.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:37.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:37.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:37.819 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:37.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-03-23T19:57:37.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:rbd mirror status : get status for rbd mirror 2026-03-23T19:57:37.868 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:admin_socket: invalid command 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ shift 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local local_cluster=cluster2:1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=local_cluster 2026-03-23T19:57:37.870 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval local_cluster=cluster2 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ local_cluster=cluster2 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:37.871 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-03-23T19:57:37.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ return 0 2026-03-23T19:57:37.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status_result=' "state": "Stopped"' 2026-03-23T19:57:37.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo ' "state": "Stopped"' 2026-03-23T19:57:37.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep -i Replaying 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test stopped = stopped 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:37.949 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped '' '' 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:57:38.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-03-23T19:57:38.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote_user: 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.0 on vm08.local 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:57:34 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster2-DNE 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:57:38 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 220 .mirror.primary.d1fc621b-fd26-4399-882d-c490a7606324.8f12d1dd-1a13-40c3-9632-622502dc12c4 (peer_uuids:[04da4782-4604-444b-8d54-7d24cb321a84])' 2026-03-23T19:57:38.998 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:57:38.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:57:38.999 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:57:39.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.0 on vm08.local 2026-03-23T19:57:39.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:57:39.003 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:57:34' 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+stopped '2026-03-23 19:57:34' 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:57:34' 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:57:39.005 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:57:39.006 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-03-23T19:57:39.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK165406546mirror.0vm08.local20.2.0-712-g70f8415btrueOK65476553mirror.1vm08.local20.2.0-712-g70f8415bfalseOK65526562mirror.2vm08.local20.2.0-712-g70f8415bfalseOK65606568mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_force_promote_userd1fc621b-fd26-4399-882d-c490a7606324up+stoppedlocal image is primary65406546mirror.0vm08.local2026-03-23 19:57:34cluster2-DNE42224bdf-e849-40ef-873a-b5e8f8e8ff02up+stoppedlocal image is primary2026-03-23 19:57:38' 2026-03-23T19:57:39.255 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:57:39.256 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-03-23T19:57:39.257 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:57:34' 2026-03-23T19:57:39.258 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-03-23T19:57:39.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:57:39.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:57:39.259 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:57:39.260 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-03-23T19:57:39.261 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local description_pattern= 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local service_pattern= 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:57:40.262 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='test_force_promote_user: 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: global_id: d1fc621b-fd26-4399-882d-c490a7606324 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: service: mirror.1 on vm08.local 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:57:38 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: peer_sites: 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: name: cluster1-DNE 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: state: up+stopped 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: description: local image is primary 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: last_update: 2026-03-23 19:57:39 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: snapshots: 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 204 .mirror.primary.d1fc621b-fd26-4399-882d-c490a7606324.dfd98ad9-1d9b-4184-ac6f-39884e55d8d5 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329]) 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: 206 .mirror.primary.d1fc621b-fd26-4399-882d-c490a7606324.cae54bcc-d664-4d70-b8c9-055cf9b45ad0 (peer_uuids:[701534f2-a033-4b7c-b240-20580a897329])' 2026-03-23T19:57:40.302 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ state: .*up+stopped' 2026-03-23T19:57:40.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: state: up+stopped 2026-03-23T19:57:40.303 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^ description: .*' 2026-03-23T19:57:40.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: description: local image is primary 2026-03-23T19:57:40.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n '' ']' 2026-03-23T19:57:40.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:57:40.304 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep '^up+' 2026-03-23T19:57:40.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:57:40.305 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep 'service: *mirror..* on ' 2026-03-23T19:57:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: service: mirror.1 on vm08.local 2026-03-23T19:57:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update 2026-03-23T19:57:40.307 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:57:38' 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-03-23 19:57:38' 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local state_pattern=up+stopped 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local 'prev_last_update=2026-03-23 19:57:38' 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local status 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ CEPH_ARGS= 2026-03-23T19:57:40.309 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-03-23T19:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ status='OKOKOK162156217mirror.0vm08.local20.2.0-712-g70f8415btrueOK62216227mirror.1vm08.local20.2.0-712-g70f8415bfalseOK62286234mirror.2vm08.local20.2.0-712-g70f8415bfalseOK62366239mirror.3vm08.local20.2.0-712-g70f8415bfalseOKtest_force_promote_userd1fc621b-fd26-4399-882d-c490a7606324up+stoppedlocal image is primary62216227mirror.1vm08.local2026-03-23 19:57:38cluster1-DNE4ac95d5e-dab6-47cf-916c-793f9541c8feup+stoppedlocal image is primary2026-03-23 19:57:39' 2026-03-23T19:57:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local last_update state 2026-03-23T19:57:40.355 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-03-23T19:57:40.356 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ last_update='2026-03-23 19:57:38' 2026-03-23T19:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-03-23T19:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ state=up+stopped 2026-03-23T19:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo up+stopped 2026-03-23T19:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ grep up+stopped 2026-03-23T19:57:40.359 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:up+stopped 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster1 mirror test_force_promote_user 100 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:57:40.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:57:40.438 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 2857.14 bytes/sec: 11 MiB/s 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local count=100 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local size= 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ size=4096 2026-03-23T19:57:40.447 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-03-23T19:57:40.488 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-03-23T19:57:40.667 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-03-23T19:57:40.668 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:elapsed: 0 ops: 100 ops/sec: 555.555 bytes/sec: 2.2 MiB/s 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster1 mirror test_force_promote_user 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:40.717 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster1 mirror test_force_promote_user 2026-03-23T19:57:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:40.718 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote_user 2026-03-23T19:57:40.939 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-23T19:57:40.947 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote_user 2026-03-23T19:57:43.301 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image_retry cluster2 mirror test_force_promote_user 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:43.308 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-03-23T19:57:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pool=mirror 2026-03-23T19:57:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local image=test_force_promote_user 2026-03-23T19:57:43.310 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-03-23T19:57:43.818 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-03-23T19:57:43.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-03-23T19:57:46.415 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ return 0 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ cleanup 0 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local error_code=0 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ set +e 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -ne 0 ']' 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z '' ']' 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster instance 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:57:46.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-03-23T19:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror' does not exist 2026-03-23T19:57:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:57:47.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-03-23T19:57:48.272 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror' does not exist 2026-03-23T19:57:48.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:57:48.284 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-03-23T19:57:49.044 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror_parent' does not exist 2026-03-23T19:57:49.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ CEPH_ARGS= 2026-03-23T19:57:49.055 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-03-23T19:57:50.385 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:pool 'mirror_parent' does not exist 2026-03-23T19:57:50.397 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-03-23T19:57:50.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster1 2026-03-23T19:57:50.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1 2026-03-23T19:57:50.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.405 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:0 '' 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:0 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:0 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:0 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:0 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:50.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:57:50.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:57:50.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:50.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:50.407 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:57:50.418 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:57:50.419 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:57:50.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=105671 2026-03-23T19:57:50.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 105671 ']' 2026-03-23T19:57:50.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 105671 2026-03-23T19:57:50.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.420 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:50.422 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.423 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105671 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:50.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.434 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105671 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:50.444 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:50.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:50.461 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:0 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:50.463 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-03-23T19:57:50.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:50.477 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster1:0 ']' 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:50.479 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-03-23T19:57:50.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:57:50.494 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:1 '' 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:1 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:50.495 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:50.496 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:57:50.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:57:50.510 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:57:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=105699 2026-03-23T19:57:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 105699 ']' 2026-03-23T19:57:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 105699 2026-03-23T19:57:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.512 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:50.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.513 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105699 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.523 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105699 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:50.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:50.552 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:1 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.554 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:50.555 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-03-23T19:57:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:50.574 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:1 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:1 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:1 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:50.575 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster1:1 ']' 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:50.576 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-03-23T19:57:50.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:57:50.590 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:2 '' 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:2 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:50.591 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:50.592 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:57:50.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:57:50.606 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:57:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=105760 2026-03-23T19:57:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 105760 ']' 2026-03-23T19:57:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 105760 2026-03-23T19:57:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.608 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:50.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.610 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105760 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.624 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:50.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.625 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105760 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:50.635 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:50.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:50.655 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:50.657 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-03-23T19:57:50.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:50.677 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:2 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:2 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:2 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster1:2 ']' 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:50.678 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:50.679 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-03-23T19:57:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:57:50.696 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster1:3 '' 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster1:3 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:50.697 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster1:3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster1:3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster1:3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster1 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster1 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:50.698 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:57:50.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:57:50.719 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:57:50.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=105837 2026-03-23T19:57:50.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 105837 ']' 2026-03-23T19:57:50.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 105837 2026-03-23T19:57:50.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.720 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:50.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.723 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105837 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.735 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=105837 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-03-23T19:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:50.766 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:50.767 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster1:3 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:50.768 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster1 2026-03-23T19:57:50.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster1 2026-03-23T19:57:50.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:50.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:50.769 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-03-23T19:57:50.785 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:50.786 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster1:3 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1:3 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster1:3 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster1 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster1:3 ']' 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster1 2026-03-23T19:57:50.787 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster1 2026-03-23T19:57:50.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:50.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:50.788 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-03-23T19:57:50.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:57:50.806 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-03-23T19:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-03-23T19:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirrors cluster2 2026-03-23T19:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2 2026-03-23T19:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ seq 0 3 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:0 '' 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:0 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:50.809 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=0 2026-03-23T19:57:50.810 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-03-23T19:57:50.827 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:57:50.828 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:57:50.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=102751 2026-03-23T19:57:50.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 102751 ']' 2026-03-23T19:57:50.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 102751 2026-03-23T19:57:50.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.829 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:50.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102751 '$2 == pid {print; exit 1}' 2026-03-23T19:57:50.833 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:50.851 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 102751 0.6 0.0 0 0 ? Rs 19:51 0:02 [rbd-mirror] 2026-03-23T19:57:50.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:50.852 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:57:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:51.854 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102751 '$2 == pid {print; exit 1}' 2026-03-23T19:57:51.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:51.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:51.864 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102751 '$2 == pid {print; exit 1}' 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:0 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:51.875 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:57:51.890 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:51.891 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:0 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:51.892 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:51.893 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-03-23T19:57:51.911 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:51.912 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:0 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:0 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:0 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=0 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 0 = cluster2:0 ']' 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=0 2026-03-23T19:57:51.913 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=0 2026-03-23T19:57:51.914 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-03-23T19:57:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:57:51.931 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-03-23T19:57:51.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:51.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:1 '' 2026-03-23T19:57:51.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:1 2026-03-23T19:57:51.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:51.932 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:1 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:1 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:1 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:57:51.933 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:57:51.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=1 2026-03-23T19:57:51.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=1 2026-03-23T19:57:51.934 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-03-23T19:57:51.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:57:51.950 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:57:51.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=102795 2026-03-23T19:57:51.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 102795 ']' 2026-03-23T19:57:51.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 102795 2026-03-23T19:57:51.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:51.952 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:51.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:51.955 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102795 '$2 == pid {print; exit 1}' 2026-03-23T19:57:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:51.968 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102795 '$2 == pid {print; exit 1}' 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:51.979 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:57:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:51.995 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:1 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-03-23T19:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:1 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:1 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:1 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=1 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 1 = cluster2:1 ']' 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=1 2026-03-23T19:57:52.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=1 2026-03-23T19:57:52.016 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-03-23T19:57:52.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:57:52.030 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:2 '' 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:2 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:52.031 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 2 = cluster2:2 ']' 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=2 2026-03-23T19:57:52.032 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-03-23T19:57:52.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:57:52.048 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=102839 2026-03-23T19:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 102839 ']' 2026-03-23T19:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 102839 2026-03-23T19:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:52.053 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:52.054 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102839 '$2 == pid {print; exit 1}' 2026-03-23T19:57:52.063 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:ubuntu 102839 0.2 0.0 0 0 ? Rs 19:51 0:00 [rbd-mirror] 2026-03-23T19:57:52.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:52.064 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 1 2026-03-23T19:57:53.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:53.066 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102839 '$2 == pid {print; exit 1}' 2026-03-23T19:57:53.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:53.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:53.076 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102839 '$2 == pid {print; exit 1}' 2026-03-23T19:57:53.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-03-23T19:57:53.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:2 2026-03-23T19:57:53.088 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:53.089 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:57:53.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:57:53.105 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:53.106 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:53.107 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-03-23T19:57:53.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:57:53.122 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-03-23T19:57:53.123 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 2 = cluster2:2 ']' 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=2 2026-03-23T19:57:53.124 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-03-23T19:57:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:57:53.139 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ stop_mirror cluster2:3 '' 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local cluster=cluster2:3 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local sig= 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test -n '' 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ local pid 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ daemon_pid_file cluster2:3 2026-03-23T19:57:53.140 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster=cluster2:3 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local val=cluster2:3 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local cluster_var_name=cluster 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local instance_var_name=instance 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ '[' 3 = cluster2:3 ']' 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval cluster=cluster2 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ cluster=cluster2 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ eval instance=3 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ instance=3 2026-03-23T19:57:53.141 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-03-23T19:57:53.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:57:53.156 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ pid=102933 2026-03-23T19:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -n 102933 ']' 2026-03-23T19:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ kill 102933 2026-03-23T19:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ for s in 0 1 2 4 8 16 32 2026-03-23T19:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ sleep 0 2026-03-23T19:57:53.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:53.160 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102933 '$2 == pid {print; exit 1}' 2026-03-23T19:57:53.170 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ break 2026-03-23T19:57:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ ps auxww 2026-03-23T19:57:53.171 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ awk -v pid=102933 '$2 == pid {print; exit 1}' 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:3 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster1 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:53.182 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:57:53.196 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:57:53.197 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local local_cluster=cluster2:3 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=local_cluster 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval local_cluster=cluster2 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ local_cluster=cluster2 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:53.198 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-03-23T19:57:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:57:53.214 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ daemon_pid_file cluster2:3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster=cluster2:3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local val=cluster2:3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local cluster_var_name=cluster 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ local instance_var_name=instance 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ cluster=cluster2 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ instance=3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ '[' 3 = cluster2:3 ']' 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval cluster=cluster2 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ cluster=cluster2 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ eval instance=3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ instance=3 2026-03-23T19:57:53.215 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-03-23T19:57:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:57:53.231 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-03-23T19:57:53.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' -z 1 ']' 2026-03-23T19:57:53.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ test '' = /tmp/tmp.ZXt03XlClS 2026-03-23T19:57:53.232 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ rm -Rf /tmp/tmp.ZXt03XlClS 2026-03-23T19:57:53.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ '[' 0 -eq 0 ']' 2026-03-23T19:57:53.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ echo OK 2026-03-23T19:57:53.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stdout:OK 2026-03-23T19:57:53.233 INFO:tasks.workunit.cluster1.client.mirror.vm08.stderr:+ exit 0 2026-03-23T19:57:53.233 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-03-23T19:57:53.233 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-03-23T19:57:53.277 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror.sh'] on cluster1.client.mirror... 2026-03-23T19:57:53.278 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-03-23T19:57:53.685 DEBUG:teuthology.parallel:result is None 2026-03-23T19:57:53.686 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-23T19:57:53.716 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-23T19:57:53.717 DEBUG:teuthology.orchestra.run.vm08:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-03-23T19:57:53.777 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-03-23T19:57:53.777 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-03-23T19:57:53.780 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-03-23T19:57:53.780 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:57:54.080 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:57:54.080 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:57:54.094 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":1317,"stamp":"2026-03-23T19:57:52.649915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":382476,"kb_used_data":13372,"kb_used_omap":4387,"kb_used_meta":364636,"kb_avail":282733044,"statfs":{"total":289910292480,"available":289518637056,"internally_reserved":0,"allocated":13692928,"data_stored":7879928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4492465,"internal_metadata":373388111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-6243635254,"num_objects":-1652,"num_object_clones":-76,"num_object_copies":-3304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1652,"num_whiteouts":0,"num_read":-102770,"num_read_kb":-13939526,"num_write":-17601,"num_write_kb":-22392913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-5,"num_bytes_recovered":-4096,"num_keys_recovered":-2,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001119"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198613+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:57:48.198613+0000","last_peered":"2026-03-23T19:57:48.198613+0000","last_clean":"2026-03-23T19:57:48.198613+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:57:48.198613+0000","last_undegraded":"2026-03-23T19:57:48.198613+0000","last_fullsized":"2026-03-23T19:57:48.198613+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198578+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:57:48.198578+0000","last_peered":"2026-03-23T19:57:48.198578+0000","last_clean":"2026-03-23T19:57:48.198578+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:57:48.198578+0000","last_undegraded":"2026-03-23T19:57:48.198578+0000","last_fullsized":"2026-03-23T19:57:48.198578+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198573+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:57:48.198573+0000","last_peered":"2026-03-23T19:57:48.198573+0000","last_clean":"2026-03-23T19:57:48.198573+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:57:48.198573+0000","last_undegraded":"2026-03-23T19:57:48.198573+0000","last_fullsized":"2026-03-23T19:57:48.198573+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198469+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:57:48.198469+0000","last_peered":"2026-03-23T19:57:48.198469+0000","last_clean":"2026-03-23T19:57:48.198469+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:57:48.198469+0000","last_undegraded":"2026-03-23T19:57:48.198469+0000","last_fullsized":"2026-03-23T19:57:48.198469+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.326141+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:57:48.326141+0000","last_peered":"2026-03-23T19:57:48.326141+0000","last_clean":"2026-03-23T19:57:48.326141+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:57:48.326141+0000","last_undegraded":"2026-03-23T19:57:48.326141+0000","last_fullsized":"2026-03-23T19:57:48.326141+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.316765+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:57:48.316765+0000","last_peered":"2026-03-23T19:57:48.316765+0000","last_clean":"2026-03-23T19:57:48.316765+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:57:48.316765+0000","last_undegraded":"2026-03-23T19:57:48.316765+0000","last_fullsized":"2026-03-23T19:57:48.316765+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.316627+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:57:48.316627+0000","last_peered":"2026-03-23T19:57:48.316627+0000","last_clean":"2026-03-23T19:57:48.316627+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:57:48.316627+0000","last_undegraded":"2026-03-23T19:57:48.316627+0000","last_fullsized":"2026-03-23T19:57:48.316627+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.197920+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:57:48.197920+0000","last_peered":"2026-03-23T19:57:48.197920+0000","last_clean":"2026-03-23T19:57:48.197920+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:57:48.197920+0000","last_undegraded":"2026-03-23T19:57:48.197920+0000","last_fullsized":"2026-03-23T19:57:48.197920+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1131,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.197875+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:57:48.197875+0000","last_peered":"2026-03-23T19:57:48.197875+0000","last_clean":"2026-03-23T19:57:48.197875+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:57:48.197875+0000","last_undegraded":"2026-03-23T19:57:48.197875+0000","last_fullsized":"2026-03-23T19:57:48.197875+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738695,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121080,"kb_used_data":4064,"kb_used_omap":1376,"kb_used_meta":115615,"kb_avail":94250760,"statfs":{"total":96636764160,"available":96512778240,"internally_reserved":0,"allocated":4161536,"data_stored":2230307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1409062,"internal_metadata":118390746},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738695,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121672,"kb_used_data":4656,"kb_used_omap":1375,"kb_used_meta":115616,"kb_avail":94250168,"statfs":{"total":96636764160,"available":96512172032,"internally_reserved":0,"allocated":4767744,"data_stored":2824820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1408837,"internal_metadata":118390971},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738694,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139724,"kb_used_data":4652,"kb_used_omap":1635,"kb_used_meta":133404,"kb_avail":94232116,"statfs":{"total":96636764160,"available":96493686784,"internally_reserved":0,"allocated":4763648,"data_stored":2824801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1674566,"internal_metadata":136606394},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:57:54.094 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:57:54.489 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:57:54.489 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:57:54.502 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":1317,"stamp":"2026-03-23T19:57:52.649915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":382476,"kb_used_data":13372,"kb_used_omap":4387,"kb_used_meta":364636,"kb_avail":282733044,"statfs":{"total":289910292480,"available":289518637056,"internally_reserved":0,"allocated":13692928,"data_stored":7879928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4492465,"internal_metadata":373388111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-6243635254,"num_objects":-1652,"num_object_clones":-76,"num_object_copies":-3304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1652,"num_whiteouts":0,"num_read":-102770,"num_read_kb":-13939526,"num_write":-17601,"num_write_kb":-22392913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-5,"num_bytes_recovered":-4096,"num_keys_recovered":-2,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001119"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198613+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:57:48.198613+0000","last_peered":"2026-03-23T19:57:48.198613+0000","last_clean":"2026-03-23T19:57:48.198613+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:57:48.198613+0000","last_undegraded":"2026-03-23T19:57:48.198613+0000","last_fullsized":"2026-03-23T19:57:48.198613+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198578+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:57:48.198578+0000","last_peered":"2026-03-23T19:57:48.198578+0000","last_clean":"2026-03-23T19:57:48.198578+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:57:48.198578+0000","last_undegraded":"2026-03-23T19:57:48.198578+0000","last_fullsized":"2026-03-23T19:57:48.198578+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198573+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:57:48.198573+0000","last_peered":"2026-03-23T19:57:48.198573+0000","last_clean":"2026-03-23T19:57:48.198573+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:57:48.198573+0000","last_undegraded":"2026-03-23T19:57:48.198573+0000","last_fullsized":"2026-03-23T19:57:48.198573+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.198469+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:57:48.198469+0000","last_peered":"2026-03-23T19:57:48.198469+0000","last_clean":"2026-03-23T19:57:48.198469+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:57:48.198469+0000","last_undegraded":"2026-03-23T19:57:48.198469+0000","last_fullsized":"2026-03-23T19:57:48.198469+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.326141+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:57:48.326141+0000","last_peered":"2026-03-23T19:57:48.326141+0000","last_clean":"2026-03-23T19:57:48.326141+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:57:48.326141+0000","last_undegraded":"2026-03-23T19:57:48.326141+0000","last_fullsized":"2026-03-23T19:57:48.326141+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1016,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.316765+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:57:48.316765+0000","last_peered":"2026-03-23T19:57:48.316765+0000","last_clean":"2026-03-23T19:57:48.316765+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:57:48.316765+0000","last_undegraded":"2026-03-23T19:57:48.316765+0000","last_fullsized":"2026-03-23T19:57:48.316765+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.316627+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:57:48.316627+0000","last_peered":"2026-03-23T19:57:48.316627+0000","last_clean":"2026-03-23T19:57:48.316627+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:57:48.316627+0000","last_undegraded":"2026-03-23T19:57:48.316627+0000","last_fullsized":"2026-03-23T19:57:48.316627+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.197920+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:57:48.197920+0000","last_peered":"2026-03-23T19:57:48.197920+0000","last_clean":"2026-03-23T19:57:48.197920+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:57:48.197920+0000","last_undegraded":"2026-03-23T19:57:48.197920+0000","last_fullsized":"2026-03-23T19:57:48.197920+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1131,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.197875+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:57:48.197875+0000","last_peered":"2026-03-23T19:57:48.197875+0000","last_clean":"2026-03-23T19:57:48.197875+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:57:48.197875+0000","last_undegraded":"2026-03-23T19:57:48.197875+0000","last_fullsized":"2026-03-23T19:57:48.197875+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738695,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121080,"kb_used_data":4064,"kb_used_omap":1376,"kb_used_meta":115615,"kb_avail":94250760,"statfs":{"total":96636764160,"available":96512778240,"internally_reserved":0,"allocated":4161536,"data_stored":2230307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1409062,"internal_metadata":118390746},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738695,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121672,"kb_used_data":4656,"kb_used_omap":1375,"kb_used_meta":115616,"kb_avail":94250168,"statfs":{"total":96636764160,"available":96512172032,"internally_reserved":0,"allocated":4767744,"data_stored":2824820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1408837,"internal_metadata":118390971},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738694,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139724,"kb_used_data":4652,"kb_used_omap":1635,"kb_used_meta":133404,"kb_avail":94232116,"statfs":{"total":96636764160,"available":96493686784,"internally_reserved":0,"allocated":4763648,"data_stored":2824801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1674566,"internal_metadata":136606394},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:57:54.503 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-03-23T19:57:54.503 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:57:54.855 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:57:54.855 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:57:54.870 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":1318,"stamp":"2026-03-23T19:57:54.650228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":154,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":382500,"kb_used_data":13396,"kb_used_omap":4392,"kb_used_meta":364631,"kb_avail":282733020,"statfs":{"total":289910292480,"available":289518612480,"internally_reserved":0,"allocated":13717504,"data_stored":7891352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498067,"internal_metadata":373382509},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-6243635254,"num_objects":-1652,"num_object_clones":-76,"num_object_copies":-3304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1652,"num_whiteouts":0,"num_read":-102770,"num_read_kb":-13939526,"num_write":-17601,"num_write_kb":-22392913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-5,"num_bytes_recovered":-4096,"num_keys_recovered":-2,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.462308"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654394+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:57:50.654394+0000","last_peered":"2026-03-23T19:57:50.654394+0000","last_clean":"2026-03-23T19:57:50.654394+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:57:50.654394+0000","last_undegraded":"2026-03-23T19:57:50.654394+0000","last_fullsized":"2026-03-23T19:57:50.654394+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654294+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:57:50.654294+0000","last_peered":"2026-03-23T19:57:50.654294+0000","last_clean":"2026-03-23T19:57:50.654294+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:57:50.654294+0000","last_undegraded":"2026-03-23T19:57:50.654294+0000","last_fullsized":"2026-03-23T19:57:50.654294+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654163+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:57:50.654163+0000","last_peered":"2026-03-23T19:57:50.654163+0000","last_clean":"2026-03-23T19:57:50.654163+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:57:50.654163+0000","last_undegraded":"2026-03-23T19:57:50.654163+0000","last_fullsized":"2026-03-23T19:57:50.654163+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654031+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:57:50.654031+0000","last_peered":"2026-03-23T19:57:50.654031+0000","last_clean":"2026-03-23T19:57:50.654031+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:57:50.654031+0000","last_undegraded":"2026-03-23T19:57:50.654031+0000","last_fullsized":"2026-03-23T19:57:50.654031+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1017,"reported_epoch":507,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.326141+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:57:48.326141+0000","last_peered":"2026-03-23T19:57:48.326141+0000","last_clean":"2026-03-23T19:57:48.326141+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:57:48.326141+0000","last_undegraded":"2026-03-23T19:57:48.326141+0000","last_fullsized":"2026-03-23T19:57:48.326141+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.321010+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:57:50.321010+0000","last_peered":"2026-03-23T19:57:50.321010+0000","last_clean":"2026-03-23T19:57:50.321010+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:57:50.321010+0000","last_undegraded":"2026-03-23T19:57:50.321010+0000","last_fullsized":"2026-03-23T19:57:50.321010+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1019,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.317038+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:57:50.317038+0000","last_peered":"2026-03-23T19:57:50.317038+0000","last_clean":"2026-03-23T19:57:50.317038+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:57:50.317038+0000","last_undegraded":"2026-03-23T19:57:50.317038+0000","last_fullsized":"2026-03-23T19:57:50.317038+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1019,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654262+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:57:50.654262+0000","last_peered":"2026-03-23T19:57:50.654262+0000","last_clean":"2026-03-23T19:57:50.654262+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:57:50.654262+0000","last_undegraded":"2026-03-23T19:57:50.654262+0000","last_fullsized":"2026-03-23T19:57:50.654262+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1133,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.653314+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:57:50.653314+0000","last_peered":"2026-03-23T19:57:50.653314+0000","last_clean":"2026-03-23T19:57:50.653314+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:57:50.653314+0000","last_undegraded":"2026-03-23T19:57:50.653314+0000","last_fullsized":"2026-03-23T19:57:50.653314+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738696,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121096,"kb_used_data":4080,"kb_used_omap":1379,"kb_used_meta":115612,"kb_avail":94250744,"statfs":{"total":96636764160,"available":96512761856,"internally_reserved":0,"allocated":4177920,"data_stored":2238082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1412972,"internal_metadata":118386836},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738696,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121680,"kb_used_data":4664,"kb_used_omap":1377,"kb_used_meta":115614,"kb_avail":94250160,"statfs":{"total":96636764160,"available":96512163840,"internally_reserved":0,"allocated":4775936,"data_stored":2828469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1410529,"internal_metadata":118389279},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738694,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139724,"kb_used_data":4652,"kb_used_omap":1635,"kb_used_meta":133404,"kb_avail":94232116,"statfs":{"total":96636764160,"available":96493686784,"internally_reserved":0,"allocated":4763648,"data_stored":2824801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1674566,"internal_metadata":136606394},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:57:54.871 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-03-23T19:57:55.105 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:57:55.105 INFO:teuthology.orchestra.run.vm08.stdout:{"epoch":508,"fsid":"60776ecd-5e60-43ff-94d1-72ba986d0b0a","created":"2026-03-23T19:30:43.981405+0000","modified":"2026-03-23T19:57:50.308627+0000","last_up_change":"2026-03-23T19:30:48.081660+0000","last_in_change":"2026-03-23T19:30:44.758967+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:48.364206+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:52.255055+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a86c779b-a4be-4ae9-878e-2afab05a4e94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6808","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6809","nonce":3072510388}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6810","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6811","nonce":3072510388}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6814","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6815","nonce":3072510388}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6812","nonce":3072510388},{"type":"v1","addr":"192.168.123.108:6813","nonce":3072510388}]},"public_addr":"192.168.123.108:6809/3072510388","cluster_addr":"192.168.123.108:6811/3072510388","heartbeat_back_addr":"192.168.123.108:6815/3072510388","heartbeat_front_addr":"192.168.123.108:6813/3072510388","state":["exists","up"]},{"osd":1,"uuid":"d124475d-dbc1-41f7-bfa1-4381afa83223","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6816","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6817","nonce":326988043}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6818","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6819","nonce":326988043}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6822","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6823","nonce":326988043}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6820","nonce":326988043},{"type":"v1","addr":"192.168.123.108:6821","nonce":326988043}]},"public_addr":"192.168.123.108:6817/326988043","cluster_addr":"192.168.123.108:6819/326988043","heartbeat_back_addr":"192.168.123.108:6823/326988043","heartbeat_front_addr":"192.168.123.108:6821/326988043","state":["exists","up"]},{"osd":2,"uuid":"c723f7e0-6e0e-4a87-8b10-c2d84cf17e6d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6800","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6801","nonce":1593614681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6802","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6803","nonce":1593614681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6806","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6807","nonce":1593614681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.108:6804","nonce":1593614681},{"type":"v1","addr":"192.168.123.108:6805","nonce":1593614681}]},"public_addr":"192.168.123.108:6801/1593614681","cluster_addr":"192.168.123.108:6803/1593614681","heartbeat_back_addr":"192.168.123.108:6807/1593614681","heartbeat_front_addr":"192.168.123.108:6805/1593614681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.330219+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.267454+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:46.318092+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":204,"length":1},{"begin":206,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:57:56.119 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-03-23T19:57:56.120 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:57:56.224 INFO:teuthology.orchestra.run.vm08.stdout:{ 2026-03-23T19:57:56.224 INFO:teuthology.orchestra.run.vm08.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-23T19:57:56.224 INFO:teuthology.orchestra.run.vm08.stdout:} 2026-03-23T19:57:56.237 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-03-23T19:57:56.519 INFO:teuthology.orchestra.run.vm08.stderr:instructed osd(s) 0 to deep-scrub 2026-03-23T19:57:56.532 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-03-23T19:57:56.532 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:57:56.630 INFO:teuthology.orchestra.run.vm08.stdout:{ 2026-03-23T19:57:56.630 INFO:teuthology.orchestra.run.vm08.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-23T19:57:56.631 INFO:teuthology.orchestra.run.vm08.stdout:} 2026-03-23T19:57:56.642 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-03-23T19:57:56.868 INFO:teuthology.orchestra.run.vm08.stderr:instructed osd(s) 1 to deep-scrub 2026-03-23T19:57:56.883 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-03-23T19:57:56.883 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:57:56.985 INFO:teuthology.orchestra.run.vm08.stdout:{ 2026-03-23T19:57:56.985 INFO:teuthology.orchestra.run.vm08.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-23T19:57:56.985 INFO:teuthology.orchestra.run.vm08.stdout:} 2026-03-23T19:57:56.995 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-03-23T19:57:57.220 INFO:teuthology.orchestra.run.vm08.stderr:instructed osd(s) 2 to deep-scrub 2026-03-23T19:57:57.234 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:57:57.463 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:57:57.463 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:57:57.476 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":1319,"stamp":"2026-03-23T19:57:56.650580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":65,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":382504,"kb_used_data":13408,"kb_used_omap":4393,"kb_used_meta":364630,"kb_avail":282733016,"statfs":{"total":289910292480,"available":289518608384,"internally_reserved":0,"allocated":13729792,"data_stored":7895020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4499393,"internal_metadata":373381183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-11506,"num_read_kb":-14004,"num_write":-942,"num_write_kb":-1942,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.458418"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654394+0000","last_change":"2026-03-23T19:30:56.129922+0000","last_active":"2026-03-23T19:57:50.654394+0000","last_peered":"2026-03-23T19:57:50.654394+0000","last_clean":"2026-03-23T19:57:50.654394+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:57:50.654394+0000","last_undegraded":"2026-03-23T19:57:50.654394+0000","last_fullsized":"2026-03-23T19:57:50.654394+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:30:41.047619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654294+0000","last_change":"2026-03-23T19:30:56.129867+0000","last_active":"2026-03-23T19:57:50.654294+0000","last_peered":"2026-03-23T19:57:50.654294+0000","last_clean":"2026-03-23T19:57:50.654294+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:57:50.654294+0000","last_undegraded":"2026-03-23T19:57:50.654294+0000","last_fullsized":"2026-03-23T19:57:50.654294+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T19:48:15.837645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654163+0000","last_change":"2026-03-23T19:30:56.130063+0000","last_active":"2026-03-23T19:57:50.654163+0000","last_peered":"2026-03-23T19:57:50.654163+0000","last_clean":"2026-03-23T19:57:50.654163+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:57:50.654163+0000","last_undegraded":"2026-03-23T19:57:50.654163+0000","last_fullsized":"2026-03-23T19:57:50.654163+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:56:38.292523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654031+0000","last_change":"2026-03-23T19:30:56.130290+0000","last_active":"2026-03-23T19:57:50.654031+0000","last_peered":"2026-03-23T19:57:50.654031+0000","last_clean":"2026-03-23T19:57:50.654031+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:57:50.654031+0000","last_undegraded":"2026-03-23T19:57:50.654031+0000","last_fullsized":"2026-03-23T19:57:50.654031+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:51:58.351271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1019,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.834244+0000","last_change":"2026-03-23T19:30:56.127375+0000","last_active":"2026-03-23T19:57:50.834244+0000","last_peered":"2026-03-23T19:57:50.834244+0000","last_clean":"2026-03-23T19:57:50.834244+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:57:50.834244+0000","last_undegraded":"2026-03-23T19:57:50.834244+0000","last_fullsized":"2026-03-23T19:57:50.834244+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:12:47.334689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1018,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.321010+0000","last_change":"2026-03-23T19:30:56.261709+0000","last_active":"2026-03-23T19:57:50.321010+0000","last_peered":"2026-03-23T19:57:50.321010+0000","last_clean":"2026-03-23T19:57:50.321010+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:57:50.321010+0000","last_undegraded":"2026-03-23T19:57:50.321010+0000","last_fullsized":"2026-03-23T19:57:50.321010+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:23.316150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1019,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.317038+0000","last_change":"2026-03-23T19:30:56.261712+0000","last_active":"2026-03-23T19:57:50.317038+0000","last_peered":"2026-03-23T19:57:50.317038+0000","last_clean":"2026-03-23T19:57:50.317038+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:57:50.317038+0000","last_undegraded":"2026-03-23T19:57:50.317038+0000","last_fullsized":"2026-03-23T19:57:50.317038+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:51:36.259406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1019,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.654262+0000","last_change":"2026-03-23T19:30:56.130100+0000","last_active":"2026-03-23T19:57:50.654262+0000","last_peered":"2026-03-23T19:57:50.654262+0000","last_clean":"2026-03-23T19:57:50.654262+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:57:50.654262+0000","last_undegraded":"2026-03-23T19:57:50.654262+0000","last_fullsized":"2026-03-23T19:57:50.654262+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:53.110490+0000","last_clean_scrub_stamp":"2026-03-23T19:30:53.110490+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:35:39.110191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1133,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:50.653314+0000","last_change":"2026-03-23T19:30:50.371967+0000","last_active":"2026-03-23T19:57:50.653314+0000","last_peered":"2026-03-23T19:57:50.653314+0000","last_clean":"2026-03-23T19:57:50.653314+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:57:50.653314+0000","last_undegraded":"2026-03-23T19:57:50.653314+0000","last_fullsized":"2026-03-23T19:57:50.653314+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:49.088309+0000","last_clean_scrub_stamp":"2026-03-23T19:30:49.088309+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:09:01.550512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738696,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121096,"kb_used_data":4080,"kb_used_omap":1379,"kb_used_meta":115612,"kb_avail":94250744,"statfs":{"total":96636764160,"available":96512761856,"internally_reserved":0,"allocated":4177920,"data_stored":2238082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1412972,"internal_metadata":118386836},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738696,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121680,"kb_used_data":4664,"kb_used_omap":1377,"kb_used_meta":115614,"kb_avail":94250160,"statfs":{"total":96636764160,"available":96512163840,"internally_reserved":0,"allocated":4775936,"data_stored":2828469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1410529,"internal_metadata":118389279},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738695,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139728,"kb_used_data":4664,"kb_used_omap":1636,"kb_used_meta":133403,"kb_avail":94232112,"statfs":{"total":96636764160,"available":96493682688,"internally_reserved":0,"allocated":4775936,"data_stored":2828469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1675892,"internal_metadata":136605068},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-03-23T19:30:53.110490+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=53, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-03-23T19:30:49.088309+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=49, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=57, tm_sec=55, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:57:57.478 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-03-23T19:58:17.478 DEBUG:teuthology.orchestra.run.vm08:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-03-23T19:58:17.696 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:58:17.697 INFO:teuthology.orchestra.run.vm08.stderr:dumped all 2026-03-23T19:58:17.711 INFO:teuthology.orchestra.run.vm08.stdout:{"pg_ready":true,"pg_map":{"version":1329,"stamp":"2026-03-23T19:58:16.653621+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":382480,"kb_used_data":13408,"kb_used_omap":4393,"kb_used_meta":364630,"kb_avail":282733040,"statfs":{"total":289910292480,"available":289518632960,"internally_reserved":0,"allocated":13729792,"data_stored":7895020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4499393,"internal_metadata":373381183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001845"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1026,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:58:02.122457+0000","last_change":"2026-03-23T19:58:02.122457+0000","last_active":"2026-03-23T19:58:02.122457+0000","last_peered":"2026-03-23T19:58:02.122457+0000","last_clean":"2026-03-23T19:58:02.122457+0000","last_became_active":"2026-03-23T19:30:54.121220+0000","last_became_peered":"2026-03-23T19:30:54.121220+0000","last_unstale":"2026-03-23T19:58:02.122457+0000","last_undegraded":"2026-03-23T19:58:02.122457+0000","last_fullsized":"2026-03-23T19:58:02.122457+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:02.122394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:02.122394+0000","last_clean_scrub_stamp":"2026-03-23T19:58:02.122394+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:23:33.825379+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.000414776,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1026,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:58:01.075261+0000","last_change":"2026-03-23T19:58:01.075261+0000","last_active":"2026-03-23T19:58:01.075261+0000","last_peered":"2026-03-23T19:58:01.075261+0000","last_clean":"2026-03-23T19:58:01.075261+0000","last_became_active":"2026-03-23T19:30:54.121340+0000","last_became_peered":"2026-03-23T19:30:54.121340+0000","last_unstale":"2026-03-23T19:58:01.075261+0000","last_undegraded":"2026-03-23T19:58:01.075261+0000","last_fullsized":"2026-03-23T19:58:01.075261+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:01.075225+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:01.075225+0000","last_clean_scrub_stamp":"2026-03-23T19:58:01.075225+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:15:02.331562+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00033998500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1027,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:58:00.110638+0000","last_change":"2026-03-23T19:58:00.110608+0000","last_active":"2026-03-23T19:58:00.110638+0000","last_peered":"2026-03-23T19:58:00.110638+0000","last_clean":"2026-03-23T19:58:00.110638+0000","last_became_active":"2026-03-23T19:30:54.121295+0000","last_became_peered":"2026-03-23T19:30:54.121295+0000","last_unstale":"2026-03-23T19:58:00.110638+0000","last_undegraded":"2026-03-23T19:58:00.110638+0000","last_fullsized":"2026-03-23T19:58:00.110638+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:00.110564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:00.110564+0000","last_clean_scrub_stamp":"2026-03-23T19:58:00.110564+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:20:25.758766+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.000441848,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1027,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:58.118581+0000","last_change":"2026-03-23T19:57:58.118556+0000","last_active":"2026-03-23T19:57:58.118581+0000","last_peered":"2026-03-23T19:57:58.118581+0000","last_clean":"2026-03-23T19:57:58.118581+0000","last_became_active":"2026-03-23T19:30:54.121251+0000","last_became_peered":"2026-03-23T19:30:54.121251+0000","last_unstale":"2026-03-23T19:57:58.118581+0000","last_undegraded":"2026-03-23T19:57:58.118581+0000","last_fullsized":"2026-03-23T19:57:58.118581+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:57:58.118517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:57:58.118517+0000","last_clean_scrub_stamp":"2026-03-23T19:57:58.118517+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:37:06.055488+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00052890000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"508'3","reported_seq":1029,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:56.599324+0000","last_change":"2026-03-23T19:57:56.599260+0000","last_active":"2026-03-23T19:57:56.599324+0000","last_peered":"2026-03-23T19:57:56.599324+0000","last_clean":"2026-03-23T19:57:56.599324+0000","last_became_active":"2026-03-23T19:30:54.123115+0000","last_became_peered":"2026-03-23T19:30:54.123115+0000","last_unstale":"2026-03-23T19:57:56.599324+0000","last_undegraded":"2026-03-23T19:57:56.599324+0000","last_fullsized":"2026-03-23T19:57:56.599324+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"508'3","last_scrub_stamp":"2026-03-23T19:57:56.599157+0000","last_deep_scrub":"508'3","last_deep_scrub_stamp":"2026-03-23T19:57:56.599157+0000","last_clean_scrub_stamp":"2026-03-23T19:57:56.599157+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:30:12.937284+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00020239800000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1027,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:57.966312+0000","last_change":"2026-03-23T19:57:57.966278+0000","last_active":"2026-03-23T19:57:57.966312+0000","last_peered":"2026-03-23T19:57:57.966312+0000","last_clean":"2026-03-23T19:57:57.966312+0000","last_became_active":"2026-03-23T19:30:54.120800+0000","last_became_peered":"2026-03-23T19:30:54.120800+0000","last_unstale":"2026-03-23T19:57:57.966312+0000","last_undegraded":"2026-03-23T19:57:57.966312+0000","last_fullsized":"2026-03-23T19:57:57.966312+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:57:57.966233+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:57:57.966233+0000","last_clean_scrub_stamp":"2026-03-23T19:57:57.966233+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:29:28.553483+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00025330399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1028,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:58.950181+0000","last_change":"2026-03-23T19:57:58.950181+0000","last_active":"2026-03-23T19:57:58.950181+0000","last_peered":"2026-03-23T19:57:58.950181+0000","last_clean":"2026-03-23T19:57:58.950181+0000","last_became_active":"2026-03-23T19:30:54.121088+0000","last_became_peered":"2026-03-23T19:30:54.121088+0000","last_unstale":"2026-03-23T19:57:58.950181+0000","last_undegraded":"2026-03-23T19:57:58.950181+0000","last_fullsized":"2026-03-23T19:57:58.950181+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:57:58.950124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:57:58.950124+0000","last_clean_scrub_stamp":"2026-03-23T19:57:58.950124+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T03:18:25.925605+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.000175429,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"508'2","reported_seq":1029,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:59.147978+0000","last_change":"2026-03-23T19:57:59.147929+0000","last_active":"2026-03-23T19:57:59.147978+0000","last_peered":"2026-03-23T19:57:59.147978+0000","last_clean":"2026-03-23T19:57:59.147978+0000","last_became_active":"2026-03-23T19:30:54.120664+0000","last_became_peered":"2026-03-23T19:30:54.120664+0000","last_unstale":"2026-03-23T19:57:59.147978+0000","last_undegraded":"2026-03-23T19:57:59.147978+0000","last_fullsized":"2026-03-23T19:57:59.147978+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"508'2","last_scrub_stamp":"2026-03-23T19:57:59.147872+0000","last_deep_scrub":"508'2","last_deep_scrub_stamp":"2026-03-23T19:57:59.147872+0000","last_clean_scrub_stamp":"2026-03-23T19:57:59.147872+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:40:49.524423+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00037234699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"508'94","reported_seq":1143,"reported_epoch":508,"state":"active+clean","last_fresh":"2026-03-23T19:57:57.137974+0000","last_change":"2026-03-23T19:57:57.137974+0000","last_active":"2026-03-23T19:57:57.137974+0000","last_peered":"2026-03-23T19:57:57.137974+0000","last_clean":"2026-03-23T19:57:57.137974+0000","last_became_active":"2026-03-23T19:30:50.371813+0000","last_became_peered":"2026-03-23T19:30:50.371813+0000","last_unstale":"2026-03-23T19:57:57.137974+0000","last_undegraded":"2026-03-23T19:57:57.137974+0000","last_fullsized":"2026-03-23T19:57:57.137974+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"508'94","last_scrub_stamp":"2026-03-23T19:57:57.137915+0000","last_deep_scrub":"508'94","last_deep_scrub_stamp":"2026-03-23T19:57:57.137915+0000","last_clean_scrub_stamp":"2026-03-23T19:57:57.137915+0000","objects_scrubbed":2,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T21:00:35.547686+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94,"ondisk_log_size":94,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738700,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121088,"kb_used_data":4080,"kb_used_omap":1379,"kb_used_meta":115612,"kb_avail":94250752,"statfs":{"total":96636764160,"available":96512770048,"internally_reserved":0,"allocated":4177920,"data_stored":2238082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1412972,"internal_metadata":118386836},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738700,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121672,"kb_used_data":4664,"kb_used_omap":1377,"kb_used_meta":115614,"kb_avail":94250168,"statfs":{"total":96636764160,"available":96512172032,"internally_reserved":0,"allocated":4775936,"data_stored":2828469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1410529,"internal_metadata":118389279},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738699,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139720,"kb_used_data":4664,"kb_used_omap":1636,"kb_used_meta":133403,"kb_avail":94232120,"statfs":{"total":96636764160,"available":96493690880,"internally_reserved":0,"allocated":4775936,"data_stored":2828469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1675892,"internal_metadata":136605068},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:17.712 DEBUG:teuthology.orchestra.run.vm08:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-03-23T19:58:18.412 INFO:teuthology.misc:Shutting down mds daemons... 2026-03-23T19:58:18.412 INFO:teuthology.misc:Shutting down osd daemons... 2026-03-23T19:58:18.412 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-03-23T19:58:18.412 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:18.467 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-03-23T19:58:18.467 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-03-23T19:58:18.467 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:18.505 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-03-23T19:58:18.520 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-03-23T19:58:18.520 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:18.596 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-03-23T19:58:18.596 INFO:teuthology.misc:Shutting down mgr daemons... 2026-03-23T19:58:18.596 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-03-23T19:58:18.596 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:21.679 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mgr.x is failed for ~0s 2026-03-23T19:58:24.698 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-03-23T19:58:24.698 INFO:teuthology.misc:Shutting down mon daemons... 2026-03-23T19:58:24.698 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-03-23T19:58:24.699 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:24.713 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-03-23T19:58:24.713 INFO:tasks.ceph:Checking cluster log for badness... 2026-03-23T19:58:24.713 DEBUG:teuthology.orchestra.run.vm08:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-23T19:58:24.787 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@vm08.local 2026-03-23T19:58:24.787 DEBUG:teuthology.orchestra.run.vm08:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-03-23T19:58:24.876 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@vm08.local 2026-03-23T19:58:24.876 DEBUG:teuthology.orchestra.run.vm08:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-03-23T19:58:24.915 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@vm08.local 2026-03-23T19:58:24.915 DEBUG:teuthology.orchestra.run.vm08:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-03-23T19:58:24.999 INFO:tasks.ceph:Archiving mon data... 2026-03-23T19:58:25.000 DEBUG:teuthology.misc:Transferring archived files from vm08:/var/lib/ceph/mon/cluster2-a to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/data/cluster2.mon.a.tgz 2026-03-23T19:58:25.000 DEBUG:teuthology.orchestra.run.vm08:> mktemp 2026-03-23T19:58:25.057 INFO:teuthology.orchestra.run.vm08.stdout:/tmp/tmp.efieMnQAFG 2026-03-23T19:58:25.057 DEBUG:teuthology.orchestra.run.vm08:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.efieMnQAFG 2026-03-23T19:58:25.202 DEBUG:teuthology.orchestra.run.vm08:> sudo chmod 0666 /tmp/tmp.efieMnQAFG 2026-03-23T19:58:25.294 DEBUG:teuthology.orchestra.remote:vm08:/tmp/tmp.efieMnQAFG is 513KB 2026-03-23T19:58:25.356 DEBUG:teuthology.orchestra.run.vm08:> rm -fr /tmp/tmp.efieMnQAFG 2026-03-23T19:58:25.372 INFO:tasks.ceph:Cleaning ceph cluster... 2026-03-23T19:58:25.372 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-03-23T19:58:25.374 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-03-23T19:58:25.438 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-03-23T19:58:25.442 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-03-23T19:58:25.442 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:58:25.672 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:25.672 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:58:25.685 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":1414,"stamp":"2026-03-23T19:58:24.835559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":366556,"kb_used_data":15364,"kb_used_omap":4243,"kb_used_meta":346860,"kb_avail":282748964,"statfs":{"total":289910292480,"available":289534939136,"internally_reserved":0,"allocated":15732736,"data_stored":8849653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4345745,"internal_metadata":355184751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001875"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263074+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:57:49.263074+0000","last_peered":"2026-03-23T19:57:49.263074+0000","last_clean":"2026-03-23T19:57:49.263074+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:57:49.263074+0000","last_undegraded":"2026-03-23T19:57:49.263074+0000","last_fullsized":"2026-03-23T19:57:49.263074+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263033+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:57:49.263033+0000","last_peered":"2026-03-23T19:57:49.263033+0000","last_clean":"2026-03-23T19:57:49.263033+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:57:49.263033+0000","last_undegraded":"2026-03-23T19:57:49.263033+0000","last_fullsized":"2026-03-23T19:57:49.263033+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262983+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:57:49.262983+0000","last_peered":"2026-03-23T19:57:49.262983+0000","last_clean":"2026-03-23T19:57:49.262983+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:57:49.262983+0000","last_undegraded":"2026-03-23T19:57:49.262983+0000","last_fullsized":"2026-03-23T19:57:49.262983+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262903+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:57:49.262903+0000","last_peered":"2026-03-23T19:57:49.262903+0000","last_clean":"2026-03-23T19:57:49.262903+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:57:49.262903+0000","last_undegraded":"2026-03-23T19:57:49.262903+0000","last_fullsized":"2026-03-23T19:57:49.262903+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.978160+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:57:48.978160+0000","last_peered":"2026-03-23T19:57:48.978160+0000","last_clean":"2026-03-23T19:57:48.978160+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:57:48.978160+0000","last_undegraded":"2026-03-23T19:57:48.978160+0000","last_fullsized":"2026-03-23T19:57:48.978160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.492191+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:57:49.492191+0000","last_peered":"2026-03-23T19:57:49.492191+0000","last_clean":"2026-03-23T19:57:49.492191+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:57:49.492191+0000","last_undegraded":"2026-03-23T19:57:49.492191+0000","last_fullsized":"2026-03-23T19:57:49.492191+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.491852+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:57:49.491852+0000","last_peered":"2026-03-23T19:57:49.491852+0000","last_clean":"2026-03-23T19:57:49.491852+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:57:49.491852+0000","last_undegraded":"2026-03-23T19:57:49.491852+0000","last_fullsized":"2026-03-23T19:57:49.491852+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1158,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262688+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:57:49.262688+0000","last_peered":"2026-03-23T19:57:49.262688+0000","last_clean":"2026-03-23T19:57:49.262688+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:57:49.262688+0000","last_undegraded":"2026-03-23T19:57:49.262688+0000","last_fullsized":"2026-03-23T19:57:49.262688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1271,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262643+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:57:49.262643+0000","last_peered":"2026-03-23T19:57:49.262643+0000","last_clean":"2026-03-23T19:57:49.262643+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:57:49.262643+0000","last_undegraded":"2026-03-23T19:57:49.262643+0000","last_fullsized":"2026-03-23T19:57:49.262643+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738705,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103772,"kb_used_data":4732,"kb_used_omap":1209,"kb_used_meta":97798,"kb_avail":94268068,"statfs":{"total":96636764160,"available":96530501632,"internally_reserved":0,"allocated":4845568,"data_stored":2556293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1238613,"internal_metadata":100145579},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738705,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":122404,"kb_used_data":5316,"kb_used_omap":1450,"kb_used_meta":115605,"kb_avail":94249436,"statfs":{"total":96636764160,"available":96511422464,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1485581,"internal_metadata":118379763},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738705,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":140380,"kb_used_data":5316,"kb_used_omap":1583,"kb_used_meta":133456,"kb_avail":94231460,"statfs":{"total":96636764160,"available":96493015040,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1621551,"internal_metadata":136659409},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:25.685 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:58:25.918 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:25.919 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:58:25.933 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":1414,"stamp":"2026-03-23T19:58:24.835559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":366556,"kb_used_data":15364,"kb_used_omap":4243,"kb_used_meta":346860,"kb_avail":282748964,"statfs":{"total":289910292480,"available":289534939136,"internally_reserved":0,"allocated":15732736,"data_stored":8849653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4345745,"internal_metadata":355184751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001875"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263074+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:57:49.263074+0000","last_peered":"2026-03-23T19:57:49.263074+0000","last_clean":"2026-03-23T19:57:49.263074+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:57:49.263074+0000","last_undegraded":"2026-03-23T19:57:49.263074+0000","last_fullsized":"2026-03-23T19:57:49.263074+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263033+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:57:49.263033+0000","last_peered":"2026-03-23T19:57:49.263033+0000","last_clean":"2026-03-23T19:57:49.263033+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:57:49.263033+0000","last_undegraded":"2026-03-23T19:57:49.263033+0000","last_fullsized":"2026-03-23T19:57:49.263033+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262983+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:57:49.262983+0000","last_peered":"2026-03-23T19:57:49.262983+0000","last_clean":"2026-03-23T19:57:49.262983+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:57:49.262983+0000","last_undegraded":"2026-03-23T19:57:49.262983+0000","last_fullsized":"2026-03-23T19:57:49.262983+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262903+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:57:49.262903+0000","last_peered":"2026-03-23T19:57:49.262903+0000","last_clean":"2026-03-23T19:57:49.262903+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:57:49.262903+0000","last_undegraded":"2026-03-23T19:57:49.262903+0000","last_fullsized":"2026-03-23T19:57:49.262903+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.978160+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:57:48.978160+0000","last_peered":"2026-03-23T19:57:48.978160+0000","last_clean":"2026-03-23T19:57:48.978160+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:57:48.978160+0000","last_undegraded":"2026-03-23T19:57:48.978160+0000","last_fullsized":"2026-03-23T19:57:48.978160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.492191+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:57:49.492191+0000","last_peered":"2026-03-23T19:57:49.492191+0000","last_clean":"2026-03-23T19:57:49.492191+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:57:49.492191+0000","last_undegraded":"2026-03-23T19:57:49.492191+0000","last_fullsized":"2026-03-23T19:57:49.492191+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.491852+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:57:49.491852+0000","last_peered":"2026-03-23T19:57:49.491852+0000","last_clean":"2026-03-23T19:57:49.491852+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:57:49.491852+0000","last_undegraded":"2026-03-23T19:57:49.491852+0000","last_fullsized":"2026-03-23T19:57:49.491852+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1158,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262688+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:57:49.262688+0000","last_peered":"2026-03-23T19:57:49.262688+0000","last_clean":"2026-03-23T19:57:49.262688+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:57:49.262688+0000","last_undegraded":"2026-03-23T19:57:49.262688+0000","last_fullsized":"2026-03-23T19:57:49.262688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1271,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262643+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:57:49.262643+0000","last_peered":"2026-03-23T19:57:49.262643+0000","last_clean":"2026-03-23T19:57:49.262643+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:57:49.262643+0000","last_undegraded":"2026-03-23T19:57:49.262643+0000","last_fullsized":"2026-03-23T19:57:49.262643+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738705,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103772,"kb_used_data":4732,"kb_used_omap":1209,"kb_used_meta":97798,"kb_avail":94268068,"statfs":{"total":96636764160,"available":96530501632,"internally_reserved":0,"allocated":4845568,"data_stored":2556293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1238613,"internal_metadata":100145579},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738705,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":122404,"kb_used_data":5316,"kb_used_omap":1450,"kb_used_meta":115605,"kb_avail":94249436,"statfs":{"total":96636764160,"available":96511422464,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1485581,"internal_metadata":118379763},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738705,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":140380,"kb_used_data":5316,"kb_used_omap":1583,"kb_used_meta":133456,"kb_avail":94231460,"statfs":{"total":96636764160,"available":96493015040,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1621551,"internal_metadata":136659409},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:25.933 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-03-23T19:58:25.934 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:58:26.151 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:26.151 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:58:26.164 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":1414,"stamp":"2026-03-23T19:58:24.835559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":366556,"kb_used_data":15364,"kb_used_omap":4243,"kb_used_meta":346860,"kb_avail":282748964,"statfs":{"total":289910292480,"available":289534939136,"internally_reserved":0,"allocated":15732736,"data_stored":8849653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4345745,"internal_metadata":355184751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001875"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263074+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:57:49.263074+0000","last_peered":"2026-03-23T19:57:49.263074+0000","last_clean":"2026-03-23T19:57:49.263074+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:57:49.263074+0000","last_undegraded":"2026-03-23T19:57:49.263074+0000","last_fullsized":"2026-03-23T19:57:49.263074+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263033+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:57:49.263033+0000","last_peered":"2026-03-23T19:57:49.263033+0000","last_clean":"2026-03-23T19:57:49.263033+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:57:49.263033+0000","last_undegraded":"2026-03-23T19:57:49.263033+0000","last_fullsized":"2026-03-23T19:57:49.263033+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262983+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:57:49.262983+0000","last_peered":"2026-03-23T19:57:49.262983+0000","last_clean":"2026-03-23T19:57:49.262983+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:57:49.262983+0000","last_undegraded":"2026-03-23T19:57:49.262983+0000","last_fullsized":"2026-03-23T19:57:49.262983+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262903+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:57:49.262903+0000","last_peered":"2026-03-23T19:57:49.262903+0000","last_clean":"2026-03-23T19:57:49.262903+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:57:49.262903+0000","last_undegraded":"2026-03-23T19:57:49.262903+0000","last_fullsized":"2026-03-23T19:57:49.262903+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.978160+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:57:48.978160+0000","last_peered":"2026-03-23T19:57:48.978160+0000","last_clean":"2026-03-23T19:57:48.978160+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:57:48.978160+0000","last_undegraded":"2026-03-23T19:57:48.978160+0000","last_fullsized":"2026-03-23T19:57:48.978160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.492191+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:57:49.492191+0000","last_peered":"2026-03-23T19:57:49.492191+0000","last_clean":"2026-03-23T19:57:49.492191+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:57:49.492191+0000","last_undegraded":"2026-03-23T19:57:49.492191+0000","last_fullsized":"2026-03-23T19:57:49.492191+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.491852+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:57:49.491852+0000","last_peered":"2026-03-23T19:57:49.491852+0000","last_clean":"2026-03-23T19:57:49.491852+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:57:49.491852+0000","last_undegraded":"2026-03-23T19:57:49.491852+0000","last_fullsized":"2026-03-23T19:57:49.491852+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1158,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262688+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:57:49.262688+0000","last_peered":"2026-03-23T19:57:49.262688+0000","last_clean":"2026-03-23T19:57:49.262688+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:57:49.262688+0000","last_undegraded":"2026-03-23T19:57:49.262688+0000","last_fullsized":"2026-03-23T19:57:49.262688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1271,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262643+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:57:49.262643+0000","last_peered":"2026-03-23T19:57:49.262643+0000","last_clean":"2026-03-23T19:57:49.262643+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:57:49.262643+0000","last_undegraded":"2026-03-23T19:57:49.262643+0000","last_fullsized":"2026-03-23T19:57:49.262643+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738705,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103772,"kb_used_data":4732,"kb_used_omap":1209,"kb_used_meta":97798,"kb_avail":94268068,"statfs":{"total":96636764160,"available":96530501632,"internally_reserved":0,"allocated":4845568,"data_stored":2556293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1238613,"internal_metadata":100145579},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738705,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":122404,"kb_used_data":5316,"kb_used_omap":1450,"kb_used_meta":115605,"kb_avail":94249436,"statfs":{"total":96636764160,"available":96511422464,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1485581,"internal_metadata":118379763},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738705,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":140380,"kb_used_data":5316,"kb_used_omap":1583,"kb_used_meta":133456,"kb_avail":94231460,"statfs":{"total":96636764160,"available":96493015040,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1621551,"internal_metadata":136659409},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:26.165 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-03-23T19:58:26.380 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:26.380 INFO:teuthology.orchestra.run.vm04.stdout:{"epoch":577,"fsid":"2127a38f-edd0-4127-84bd-b60d939c2ed6","created":"2026-03-23T19:30:19.959664+0000","modified":"2026-03-23T19:57:48.970650+0000","last_up_change":"2026-03-23T19:30:23.994669+0000","last_in_change":"2026-03-23T19:30:20.724262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-03-23T19:30:24.542060+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-03-23T19:30:28.044741+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4ddb3b53-149c-4477-8cf0-20b408f9c667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6816","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6817","nonce":1787023239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6818","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6819","nonce":1787023239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6822","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6823","nonce":1787023239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6820","nonce":1787023239},{"type":"v1","addr":"192.168.123.104:6821","nonce":1787023239}]},"public_addr":"192.168.123.104:6817/1787023239","cluster_addr":"192.168.123.104:6819/1787023239","heartbeat_back_addr":"192.168.123.104:6823/1787023239","heartbeat_front_addr":"192.168.123.104:6821/1787023239","state":["exists","up"]},{"osd":1,"uuid":"905f6bd4-a644-41c0-8cb6-d3acdd71708a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6800","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6801","nonce":3959752833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6802","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6803","nonce":3959752833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6806","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6807","nonce":3959752833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6804","nonce":3959752833},{"type":"v1","addr":"192.168.123.104:6805","nonce":3959752833}]},"public_addr":"192.168.123.104:6801/3959752833","cluster_addr":"192.168.123.104:6803/3959752833","heartbeat_back_addr":"192.168.123.104:6807/3959752833","heartbeat_front_addr":"192.168.123.104:6805/3959752833","state":["exists","up"]},{"osd":2,"uuid":"630c2e98-f3f7-4b17-824c-cc35ddbeb468","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6808","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6809","nonce":1186798788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6810","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6811","nonce":1186798788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6814","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6815","nonce":1186798788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"192.168.123.104:6812","nonce":1186798788},{"type":"v1","addr":"192.168.123.104:6813","nonce":1186798788}]},"public_addr":"192.168.123.104:6809/1186798788","cluster_addr":"192.168.123.104:6811/1186798788","heartbeat_back_addr":"192.168.123.104:6815/1186798788","heartbeat_front_addr":"192.168.123.104:6813/1186798788","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.258841+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.196343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-03-23T19:30:22.257168+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"192.168.123.108:0/182013363":"2026-03-23T20:57:25.705215+0000","192.168.123.108:0/2080735060":"2026-03-23T20:53:08.445849+0000","192.168.123.108:0/567470730":"2026-03-23T20:53:08.811179+0000","192.168.123.108:0/2241108592":"2026-03-23T20:53:39.281356+0000","192.168.123.108:0/2607098768":"2026-03-23T20:53:39.281817+0000","192.168.123.108:0/525538457":"2026-03-23T20:53:39.280475+0000","192.168.123.108:0/1898424776":"2026-03-23T20:53:39.280389+0000","192.168.123.108:0/3636733408":"2026-03-23T20:53:39.281045+0000","192.168.123.108:0/3154472389":"2026-03-23T20:56:54.745257+0000","192.168.123.108:0/3869608695":"2026-03-23T20:53:39.280959+0000","192.168.123.108:0/4222844650":"2026-03-23T20:57:25.705080+0000","192.168.123.108:0/3625028279":"2026-03-23T20:56:48.139130+0000","192.168.123.108:0/713521818":"2026-03-23T20:57:25.703726+0000","192.168.123.108:0/1328152105":"2026-03-23T20:56:54.786058+0000","192.168.123.108:0/1613612293":"2026-03-23T20:57:25.704956+0000","192.168.123.108:0/2351604087":"2026-03-23T20:57:25.704224+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":217,"length":1},{"begin":220,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-03-23T19:58:26.679 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mgr.x has been restored 2026-03-23T19:58:27.394 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-03-23T19:58:27.394 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:58:27.496 INFO:teuthology.orchestra.run.vm04.stdout:{ 2026-03-23T19:58:27.496 INFO:teuthology.orchestra.run.vm04.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-03-23T19:58:27.496 INFO:teuthology.orchestra.run.vm04.stdout:} 2026-03-23T19:58:27.506 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-03-23T19:58:27.721 INFO:teuthology.orchestra.run.vm04.stderr:instructed osd(s) 0 to deep-scrub 2026-03-23T19:58:27.736 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-03-23T19:58:27.736 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:58:27.836 INFO:teuthology.orchestra.run.vm04.stdout:{ 2026-03-23T19:58:27.837 INFO:teuthology.orchestra.run.vm04.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-23T19:58:27.837 INFO:teuthology.orchestra.run.vm04.stdout:} 2026-03-23T19:58:27.846 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-03-23T19:58:28.061 INFO:teuthology.orchestra.run.vm04.stderr:instructed osd(s) 1 to deep-scrub 2026-03-23T19:58:28.072 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-03-23T19:58:28.075 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-03-23T19:58:28.158 INFO:teuthology.orchestra.run.vm04.stdout:{ 2026-03-23T19:58:28.158 INFO:teuthology.orchestra.run.vm04.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-03-23T19:58:28.158 INFO:teuthology.orchestra.run.vm04.stdout:} 2026-03-23T19:58:28.166 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-03-23T19:58:28.370 INFO:teuthology.orchestra.run.vm04.stderr:instructed osd(s) 2 to deep-scrub 2026-03-23T19:58:28.383 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:58:28.595 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:28.595 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:58:28.607 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":1415,"stamp":"2026-03-23T19:58:26.835866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95,"ondisk_log_size":95,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":366556,"kb_used_data":15364,"kb_used_omap":4243,"kb_used_meta":346860,"kb_avail":282748964,"statfs":{"total":289910292480,"available":289534939136,"internally_reserved":0,"allocated":15732736,"data_stored":8849653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4345745,"internal_metadata":355184751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001884"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263074+0000","last_change":"2026-03-23T19:30:32.032514+0000","last_active":"2026-03-23T19:57:49.263074+0000","last_peered":"2026-03-23T19:57:49.263074+0000","last_clean":"2026-03-23T19:57:49.263074+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:57:49.263074+0000","last_undegraded":"2026-03-23T19:57:49.263074+0000","last_fullsized":"2026-03-23T19:57:49.263074+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:47:09.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1156,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.263033+0000","last_change":"2026-03-23T19:30:32.032922+0000","last_active":"2026-03-23T19:57:49.263033+0000","last_peered":"2026-03-23T19:57:49.263033+0000","last_clean":"2026-03-23T19:57:49.263033+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:57:49.263033+0000","last_undegraded":"2026-03-23T19:57:49.263033+0000","last_fullsized":"2026-03-23T19:57:49.263033+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:47:16.158924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262983+0000","last_change":"2026-03-23T19:30:32.033080+0000","last_active":"2026-03-23T19:57:49.262983+0000","last_peered":"2026-03-23T19:57:49.262983+0000","last_clean":"2026-03-23T19:57:49.262983+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:57:49.262983+0000","last_undegraded":"2026-03-23T19:57:49.262983+0000","last_fullsized":"2026-03-23T19:57:49.262983+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:20:30.991751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262903+0000","last_change":"2026-03-23T19:30:32.033116+0000","last_active":"2026-03-23T19:57:49.262903+0000","last_peered":"2026-03-23T19:57:49.262903+0000","last_clean":"2026-03-23T19:57:49.262903+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:57:49.262903+0000","last_undegraded":"2026-03-23T19:57:49.262903+0000","last_fullsized":"2026-03-23T19:57:49.262903+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T06:21:43.965880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:48.978160+0000","last_change":"2026-03-23T19:30:32.037291+0000","last_active":"2026-03-23T19:57:48.978160+0000","last_peered":"2026-03-23T19:57:48.978160+0000","last_clean":"2026-03-23T19:57:48.978160+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:57:48.978160+0000","last_undegraded":"2026-03-23T19:57:48.978160+0000","last_fullsized":"2026-03-23T19:57:48.978160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T01:48:47.220462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.492191+0000","last_change":"2026-03-23T19:30:32.203529+0000","last_active":"2026-03-23T19:57:49.492191+0000","last_peered":"2026-03-23T19:57:49.492191+0000","last_clean":"2026-03-23T19:57:49.492191+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:57:49.492191+0000","last_undegraded":"2026-03-23T19:57:49.492191+0000","last_fullsized":"2026-03-23T19:57:49.492191+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:40:15.332483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1157,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.491852+0000","last_change":"2026-03-23T19:30:32.203580+0000","last_active":"2026-03-23T19:57:49.491852+0000","last_peered":"2026-03-23T19:57:49.491852+0000","last_clean":"2026-03-23T19:57:49.491852+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:57:49.491852+0000","last_undegraded":"2026-03-23T19:57:49.491852+0000","last_fullsized":"2026-03-23T19:57:49.491852+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T23:56:54.423998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1158,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262688+0000","last_change":"2026-03-23T19:30:32.032781+0000","last_active":"2026-03-23T19:57:49.262688+0000","last_peered":"2026-03-23T19:57:49.262688+0000","last_clean":"2026-03-23T19:57:49.262688+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:57:49.262688+0000","last_undegraded":"2026-03-23T19:57:49.262688+0000","last_fullsized":"2026-03-23T19:57:49.262688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:29.017919+0000","last_clean_scrub_stamp":"2026-03-23T19:30:29.017919+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:39:06.504876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'92","reported_seq":1271,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:57:49.262643+0000","last_change":"2026-03-23T19:30:26.341345+0000","last_active":"2026-03-23T19:57:49.262643+0000","last_peered":"2026-03-23T19:57:49.262643+0000","last_clean":"2026-03-23T19:57:49.262643+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:57:49.262643+0000","last_undegraded":"2026-03-23T19:57:49.262643+0000","last_fullsized":"2026-03-23T19:57:49.262643+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:30:25.001110+0000","last_clean_scrub_stamp":"2026-03-23T19:30:25.001110+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T02:25:20.984572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738706,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103772,"kb_used_data":4732,"kb_used_omap":1209,"kb_used_meta":97798,"kb_avail":94268068,"statfs":{"total":96636764160,"available":96530501632,"internally_reserved":0,"allocated":4845568,"data_stored":2556293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1238613,"internal_metadata":100145579},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738706,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":122404,"kb_used_data":5316,"kb_used_omap":1450,"kb_used_meta":115605,"kb_avail":94249436,"statfs":{"total":96636764160,"available":96511422464,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1485581,"internal_metadata":118379763},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738706,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":140380,"kb_used_data":5316,"kb_used_omap":1583,"kb_used_meta":133456,"kb_avail":94231460,"statfs":{"total":96636764160,"available":96493015040,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1621551,"internal_metadata":136659409},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.607 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.608 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.608 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-03-23T19:30:29.017919+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=29, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.608 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-03-23T19:30:25.001110+0000 time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=30, tm_sec=25, tm_wday=0, tm_yday=82, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=3, tm_mday=23, tm_hour=19, tm_min=58, tm_sec=26, tm_wday=0, tm_yday=82, tm_isdst=0) 2026-03-23T19:58:28.608 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-03-23T19:58:48.609 DEBUG:teuthology.orchestra.run.vm04:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-03-23T19:58:48.821 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:58:48.821 INFO:teuthology.orchestra.run.vm04.stderr:dumped all 2026-03-23T19:58:48.833 INFO:teuthology.orchestra.run.vm04.stdout:{"pg_ready":true,"pg_map":{"version":1425,"stamp":"2026-03-23T19:58:46.838767+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":125,"num_write_kb":2152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":366516,"kb_used_data":15364,"kb_used_omap":4243,"kb_used_meta":346860,"kb_avail":282749004,"statfs":{"total":289910292480,"available":289534980096,"internally_reserved":0,"allocated":15732736,"data_stored":8849653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4345745,"internal_metadata":355184751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001650"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1165,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:30.316860+0000","last_change":"2026-03-23T19:58:30.316815+0000","last_active":"2026-03-23T19:58:30.316860+0000","last_peered":"2026-03-23T19:58:30.316860+0000","last_clean":"2026-03-23T19:58:30.316860+0000","last_became_active":"2026-03-23T19:30:30.029130+0000","last_became_peered":"2026-03-23T19:30:30.029130+0000","last_unstale":"2026-03-23T19:58:30.316860+0000","last_undegraded":"2026-03-23T19:58:30.316860+0000","last_fullsized":"2026-03-23T19:58:30.316860+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:30.316751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:30.316751+0000","last_clean_scrub_stamp":"2026-03-23T19:58:30.316751+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T05:25:54.228772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000358542,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1165,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:32.292950+0000","last_change":"2026-03-23T19:58:32.292924+0000","last_active":"2026-03-23T19:58:32.292950+0000","last_peered":"2026-03-23T19:58:32.292950+0000","last_clean":"2026-03-23T19:58:32.292950+0000","last_became_active":"2026-03-23T19:30:30.030019+0000","last_became_peered":"2026-03-23T19:30:30.030019+0000","last_unstale":"2026-03-23T19:58:32.292950+0000","last_undegraded":"2026-03-23T19:58:32.292950+0000","last_fullsized":"2026-03-23T19:58:32.292950+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:32.292888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:32.292888+0000","last_clean_scrub_stamp":"2026-03-23T19:58:32.292888+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T20:23:53.420647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068923200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1165,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:31.308798+0000","last_change":"2026-03-23T19:58:31.308798+0000","last_active":"2026-03-23T19:58:31.308798+0000","last_peered":"2026-03-23T19:58:31.308798+0000","last_clean":"2026-03-23T19:58:31.308798+0000","last_became_active":"2026-03-23T19:30:30.028526+0000","last_became_peered":"2026-03-23T19:30:30.028526+0000","last_unstale":"2026-03-23T19:58:31.308798+0000","last_undegraded":"2026-03-23T19:58:31.308798+0000","last_fullsized":"2026-03-23T19:58:31.308798+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:31.308729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:31.308729+0000","last_clean_scrub_stamp":"2026-03-23T19:58:31.308729+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T04:57:14.411455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092889000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1165,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:33.334079+0000","last_change":"2026-03-23T19:58:33.334079+0000","last_active":"2026-03-23T19:58:33.334079+0000","last_peered":"2026-03-23T19:58:33.334079+0000","last_clean":"2026-03-23T19:58:33.334079+0000","last_became_active":"2026-03-23T19:30:30.028458+0000","last_became_peered":"2026-03-23T19:30:30.028458+0000","last_unstale":"2026-03-23T19:58:33.334079+0000","last_undegraded":"2026-03-23T19:58:33.334079+0000","last_fullsized":"2026-03-23T19:58:33.334079+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:33.334023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:33.334023+0000","last_clean_scrub_stamp":"2026-03-23T19:58:33.334023+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T07:46:26.871196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080730099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"577'3","reported_seq":1166,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:28.400059+0000","last_change":"2026-03-23T19:58:28.400059+0000","last_active":"2026-03-23T19:58:28.400059+0000","last_peered":"2026-03-23T19:58:28.400059+0000","last_clean":"2026-03-23T19:58:28.400059+0000","last_became_active":"2026-03-23T19:30:30.029696+0000","last_became_peered":"2026-03-23T19:30:30.029696+0000","last_unstale":"2026-03-23T19:58:28.400059+0000","last_undegraded":"2026-03-23T19:58:28.400059+0000","last_fullsized":"2026-03-23T19:58:28.400059+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"577'3","last_scrub_stamp":"2026-03-23T19:58:28.400018+0000","last_deep_scrub":"577'3","last_deep_scrub_stamp":"2026-03-23T19:58:28.400018+0000","last_clean_scrub_stamp":"2026-03-23T19:58:28.400018+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:50:30.621383+0000","scrub_duration":23,"objects_trimmed":0,"snaptrim_duration":0.000451636,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1165,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:29.896619+0000","last_change":"2026-03-23T19:58:29.896619+0000","last_active":"2026-03-23T19:58:29.896619+0000","last_peered":"2026-03-23T19:58:29.896619+0000","last_clean":"2026-03-23T19:58:29.896619+0000","last_became_active":"2026-03-23T19:30:30.029629+0000","last_became_peered":"2026-03-23T19:30:30.029629+0000","last_unstale":"2026-03-23T19:58:29.896619+0000","last_undegraded":"2026-03-23T19:58:29.896619+0000","last_fullsized":"2026-03-23T19:58:29.896619+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:29.896565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:29.896565+0000","last_clean_scrub_stamp":"2026-03-23T19:58:29.896565+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:09:44.306459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027000599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1166,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:28.912299+0000","last_change":"2026-03-23T19:58:28.912269+0000","last_active":"2026-03-23T19:58:28.912299+0000","last_peered":"2026-03-23T19:58:28.912299+0000","last_clean":"2026-03-23T19:58:28.912299+0000","last_became_active":"2026-03-23T19:30:30.029254+0000","last_became_peered":"2026-03-23T19:30:30.029254+0000","last_unstale":"2026-03-23T19:58:28.912299+0000","last_undegraded":"2026-03-23T19:58:28.912299+0000","last_fullsized":"2026-03-23T19:58:28.912299+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-03-23T19:58:28.912226+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-03-23T19:58:28.912226+0000","last_clean_scrub_stamp":"2026-03-23T19:58:28.912226+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-24T22:24:27.942331+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00024601100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"577'2","reported_seq":1167,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:29.305279+0000","last_change":"2026-03-23T19:58:29.305279+0000","last_active":"2026-03-23T19:58:29.305279+0000","last_peered":"2026-03-23T19:58:29.305279+0000","last_clean":"2026-03-23T19:58:29.305279+0000","last_became_active":"2026-03-23T19:30:30.028315+0000","last_became_peered":"2026-03-23T19:30:30.028315+0000","last_unstale":"2026-03-23T19:58:29.305279+0000","last_undegraded":"2026-03-23T19:58:29.305279+0000","last_fullsized":"2026-03-23T19:58:29.305279+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"577'2","last_scrub_stamp":"2026-03-23T19:58:29.305228+0000","last_deep_scrub":"577'2","last_deep_scrub_stamp":"2026-03-23T19:58:29.305228+0000","last_clean_scrub_stamp":"2026-03-23T19:58:29.305228+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:39:21.769933+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00057635,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"577'94","reported_seq":1282,"reported_epoch":577,"state":"active+clean","last_fresh":"2026-03-23T19:58:28.322075+0000","last_change":"2026-03-23T19:58:28.322043+0000","last_active":"2026-03-23T19:58:28.322075+0000","last_peered":"2026-03-23T19:58:28.322075+0000","last_clean":"2026-03-23T19:58:28.322075+0000","last_became_active":"2026-03-23T19:30:26.341205+0000","last_became_peered":"2026-03-23T19:30:26.341205+0000","last_unstale":"2026-03-23T19:58:28.322075+0000","last_undegraded":"2026-03-23T19:58:28.322075+0000","last_fullsized":"2026-03-23T19:58:28.322075+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"577'94","last_scrub_stamp":"2026-03-23T19:58:28.321999+0000","last_deep_scrub":"577'94","last_deep_scrub_stamp":"2026-03-23T19:58:28.321999+0000","last_clean_scrub_stamp":"2026-03-23T19:58:28.321999+0000","objects_scrubbed":2,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-03-25T00:48:34.690418+0000","scrub_duration":35,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":76,"num_read_kb":64,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94,"ondisk_log_size":94,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738710,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103756,"kb_used_data":4732,"kb_used_omap":1209,"kb_used_meta":97798,"kb_avail":94268084,"statfs":{"total":96636764160,"available":96530518016,"internally_reserved":0,"allocated":4845568,"data_stored":2556293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1238613,"internal_metadata":100145579},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738710,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":122388,"kb_used_data":5316,"kb_used_omap":1450,"kb_used_meta":115605,"kb_avail":94249452,"statfs":{"total":96636764160,"available":96511438848,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1485581,"internal_metadata":118379763},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738710,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":140372,"kb_used_data":5316,"kb_used_omap":1583,"kb_used_meta":133456,"kb_avail":94231468,"statfs":{"total":96636764160,"available":96493023232,"internally_reserved":0,"allocated":5443584,"data_stored":3146680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1621551,"internal_metadata":136659409},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-03-23T19:58:48.834 DEBUG:teuthology.orchestra.run.vm04:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-03-23T19:58:49.078 INFO:teuthology.misc:Shutting down mds daemons... 2026-03-23T19:58:49.079 INFO:teuthology.misc:Shutting down osd daemons... 2026-03-23T19:58:49.079 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-03-23T19:58:49.079 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:49.127 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-03-23T19:58:49.127 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-03-23T19:58:49.127 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:49.164 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-03-23T19:58:49.164 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-03-23T19:58:49.164 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:49.241 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-03-23T19:58:49.241 INFO:teuthology.misc:Shutting down mgr daemons... 2026-03-23T19:58:49.241 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-03-23T19:58:49.241 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:49.318 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-03-23T19:58:49.319 INFO:teuthology.misc:Shutting down mon daemons... 2026-03-23T19:58:49.319 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-03-23T19:58:49.319 INFO:teuthology.orchestra.run:waiting for 300 2026-03-23T19:58:49.371 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-03-23T19:58:49.371 INFO:tasks.ceph:Checking cluster log for badness... 2026-03-23T19:58:49.371 DEBUG:teuthology.orchestra.run.vm04:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-03-23T19:58:49.442 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@vm04.local 2026-03-23T19:58:49.442 DEBUG:teuthology.orchestra.run.vm04:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-03-23T19:58:49.532 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@vm04.local 2026-03-23T19:58:49.532 DEBUG:teuthology.orchestra.run.vm04:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-03-23T19:58:49.573 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@vm04.local 2026-03-23T19:58:49.573 DEBUG:teuthology.orchestra.run.vm04:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-03-23T19:58:49.652 INFO:tasks.ceph:Archiving mon data... 2026-03-23T19:58:49.652 DEBUG:teuthology.misc:Transferring archived files from vm04:/var/lib/ceph/mon/cluster1-a to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/data/cluster1.mon.a.tgz 2026-03-23T19:58:49.653 DEBUG:teuthology.orchestra.run.vm04:> mktemp 2026-03-23T19:58:49.712 INFO:teuthology.orchestra.run.vm04.stdout:/tmp/tmp.D9oAjg2h3v 2026-03-23T19:58:49.712 DEBUG:teuthology.orchestra.run.vm04:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.D9oAjg2h3v 2026-03-23T19:58:49.857 DEBUG:teuthology.orchestra.run.vm04:> sudo chmod 0666 /tmp/tmp.D9oAjg2h3v 2026-03-23T19:58:49.938 DEBUG:teuthology.orchestra.remote:vm04:/tmp/tmp.D9oAjg2h3v is 525KB 2026-03-23T19:58:49.996 DEBUG:teuthology.orchestra.run.vm04:> rm -fr /tmp/tmp.D9oAjg2h3v 2026-03-23T19:58:50.012 INFO:tasks.ceph:Cleaning ceph cluster... 2026-03-23T19:58:50.012 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-03-23T19:58:50.054 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-03-23T19:58:50.134 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-03-23T19:58:50.150 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-03-23T19:58:50.150 INFO:tasks.ceph:Archiving crash dumps... 2026-03-23T19:58:50.150 DEBUG:teuthology.misc:Transferring archived files from vm04:/var/lib/ceph/crash to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm04/crash 2026-03-23T19:58:50.151 DEBUG:teuthology.orchestra.run.vm04:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-03-23T19:58:50.203 DEBUG:teuthology.misc:Transferring archived files from vm08:/var/lib/ceph/crash to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm08/crash 2026-03-23T19:58:50.203 DEBUG:teuthology.orchestra.run.vm08:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-03-23T19:58:50.232 INFO:tasks.ceph:Compressing logs... 2026-03-23T19:58:50.232 DEBUG:teuthology.orchestra.run.vm04:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-03-23T19:58:50.246 DEBUG:teuthology.orchestra.run.vm08:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-03-23T19:58:50.271 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.51712.log 2026-03-23T19:58:50.272 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-03-23T19:58:50.272 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/ceph.tmp-client.admin.51712.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-03-23T19:58:50.272 INFO:teuthology.orchestra.run.vm04.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.51712.log.gz 2026-03-23T19:58:50.272 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-03-23T19:58:50.273 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-03-23T19:58:50.273 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-osd.1.log: /var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-03-23T19:58:50.287 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56614.log 2026-03-23T19:58:50.289 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1.log: 87.0% -- replaced with /var/log/ceph/cluster1.log.gz 2026-03-23T19:58:50.302 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56709.log 2026-03-23T19:58:50.302 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.56614.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.56614.log.gz 2026-03-23T19:58:50.303 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.54816.log 2026-03-23T19:58:50.303 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-03-23T19:58:50.304 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/ceph.tmp-client.admin.54816.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.54816.log.gz 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:gzip/var/log/ceph/cluster2-osd.2.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.60014.log 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-03-23T19:58:50.305 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60014.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60014.log.gz 2026-03-23T19:58:50.308 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60100.log 2026-03-23T19:58:50.309 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-03-23T19:58:50.309 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.56709.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.56709.log.gz 2026-03-23T19:58:50.314 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2.log: 87.0% -- replaced with /var/log/ceph/cluster2.log.gz 2026-03-23T19:58:50.317 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56854.log 2026-03-23T19:58:50.324 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60222.log 2026-03-23T19:58:50.324 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60100.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60100.log.gz 2026-03-23T19:58:50.325 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56901.log 2026-03-23T19:58:50.325 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.56854.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.56854.log.gz 2026-03-23T19:58:50.337 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56948.log 2026-03-23T19:58:50.337 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.56901.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.56901.log.gz 2026-03-23T19:58:50.339 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60269.log 2026-03-23T19:58:50.339 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60222.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60222.log.gz 2026-03-23T19:58:50.351 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57075.log 2026-03-23T19:58:50.351 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.56948.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.56948.log.gz 2026-03-23T19:58:50.355 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60316.log 2026-03-23T19:58:50.355 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60269.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60269.log.gz 2026-03-23T19:58:50.362 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60442.log 2026-03-23T19:58:50.362 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60316.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60316.log.gz 2026-03-23T19:58:50.365 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-03-23T19:58:50.365 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.57075.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.57075.log.gz 2026-03-23T19:58:50.372 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-03-23T19:58:50.372 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.60442.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60442.log.gz 2026-03-23T19:58:50.379 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61723.log 2026-03-23T19:58:50.379 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58911.log 2026-03-23T19:58:50.386 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1.audit.log: 93.7% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-03-23T19:58:50.387 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59707.log 2026-03-23T19:58:50.387 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.58911.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.58911.log.gz 2026-03-23T19:58:50.388 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62986.log 2026-03-23T19:58:50.388 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.61723.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61723.log.gz 2026-03-23T19:58:50.394 INFO:teuthology.orchestra.run.vm08.stderr: 93.9% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-03-23T19:58:50.397 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63056.log 2026-03-23T19:58:50.397 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.62986.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62986.log.gz 2026-03-23T19:58:50.403 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59777.log 2026-03-23T19:58:50.404 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59707.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59707.log.gz 2026-03-23T19:58:50.407 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63081.log 2026-03-23T19:58:50.407 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63056.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63056.log.gz 2026-03-23T19:58:50.416 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63146.log 2026-03-23T19:58:50.417 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63081.log: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.63081.log.gz 2026-03-23T19:58:50.417 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59802.log 2026-03-23T19:58:50.418 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59777.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59777.log.gz 2026-03-23T19:58:50.423 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63195.log 2026-03-23T19:58:50.424 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63146.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63146.log.gz 2026-03-23T19:58:50.424 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59867.log 2026-03-23T19:58:50.426 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59802.log: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.59802.log.gz 2026-03-23T19:58:50.431 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59916.log 2026-03-23T19:58:50.431 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59867.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59867.log.gz 2026-03-23T19:58:50.432 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63244.log 2026-03-23T19:58:50.433 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63195.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63195.log.gz 2026-03-23T19:58:50.439 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59965.log 2026-03-23T19:58:50.440 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63293.log 2026-03-23T19:58:50.440 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63244.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63244.log.gz 2026-03-23T19:58:50.440 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59916.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59916.log.gz 2026-03-23T19:58:50.446 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60014.log 2026-03-23T19:58:50.446 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.59965.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59965.log.gz 2026-03-23T19:58:50.447 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63395.log 2026-03-23T19:58:50.447 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63293.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63293.log.gz 2026-03-23T19:58:50.454 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63394.log 2026-03-23T19:58:50.454 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60117.log 2026-03-23T19:58:50.454 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63395.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63395.log.gz 2026-03-23T19:58:50.455 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60014.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60014.log.gz 2026-03-23T19:58:50.461 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60114.log 2026-03-23T19:58:50.461 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60117.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60117.log.gz 2026-03-23T19:58:50.462 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63396.log 2026-03-23T19:58:50.462 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63394.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63394.log.gz 2026-03-23T19:58:50.469 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63489.log 2026-03-23T19:58:50.469 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63396.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63396.log.gz 2026-03-23T19:58:50.470 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60116.log 2026-03-23T19:58:50.470 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60114.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60114.log.gz 2026-03-23T19:58:50.476 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63546.log 2026-03-23T19:58:50.476 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63489.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63489.log.gz 2026-03-23T19:58:50.477 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60210.log 2026-03-23T19:58:50.477 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60116.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60116.log.gz 2026-03-23T19:58:50.484 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60265.log 2026-03-23T19:58:50.486 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60210.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60210.log.gz 2026-03-23T19:58:50.486 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63547.log 2026-03-23T19:58:50.487 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63546.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63546.log.gz 2026-03-23T19:58:50.494 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63636.log 2026-03-23T19:58:50.495 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63547.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63547.log.gz 2026-03-23T19:58:50.495 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60264.log 2026-03-23T19:58:50.495 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60265.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60265.log.gz 2026-03-23T19:58:50.503 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60357.log 2026-03-23T19:58:50.503 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63683.log 2026-03-23T19:58:50.504 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63636.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63636.log.gz 2026-03-23T19:58:50.504 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60264.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60264.log.gz 2026-03-23T19:58:50.510 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63710.log 2026-03-23T19:58:50.510 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63683.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63683.log.gz 2026-03-23T19:58:50.512 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60384.log 2026-03-23T19:58:50.512 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60357.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60357.log.gz 2026-03-23T19:58:50.520 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63783.log 2026-03-23T19:58:50.520 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63710.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63710.log.gz 2026-03-23T19:58:50.522 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60441.log 2026-03-23T19:58:50.522 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60384.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60384.log.gz 2026-03-23T19:58:50.527 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60504.log 2026-03-23T19:58:50.527 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63830.log 2026-03-23T19:58:50.527 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63783.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63783.log.gz 2026-03-23T19:58:50.530 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60441.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60441.log.gz 2026-03-23T19:58:50.534 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60531.log 2026-03-23T19:58:50.534 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60504.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60504.log.gz 2026-03-23T19:58:50.535 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63857.log 2026-03-23T19:58:50.535 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63830.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63830.log.gz 2026-03-23T19:58:50.542 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63930.log 2026-03-23T19:58:50.542 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63857.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63857.log.gz 2026-03-23T19:58:50.544 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60580.log 2026-03-23T19:58:50.544 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60531.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60531.log.gz 2026-03-23T19:58:50.551 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63979.log 2026-03-23T19:58:50.552 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63930.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63930.log.gz 2026-03-23T19:58:50.552 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60651.log 2026-03-23T19:58:50.552 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60580.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60580.log.gz 2026-03-23T19:58:50.559 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60700.log 2026-03-23T19:58:50.559 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60651.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60651.log.gz 2026-03-23T19:58:50.561 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64028.log 2026-03-23T19:58:50.562 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.63979.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.63979.log.gz 2026-03-23T19:58:50.566 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60749.log 2026-03-23T19:58:50.566 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64077.log 2026-03-23T19:58:50.566 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60700.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60700.log.gz 2026-03-23T19:58:50.567 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64028.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64028.log.gz 2026-03-23T19:58:50.573 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64126.log 2026-03-23T19:58:50.573 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64077.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64077.log.gz 2026-03-23T19:58:50.574 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-03-23T19:58:50.574 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.60749.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60749.log.gz 2026-03-23T19:58:50.580 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64175.log 2026-03-23T19:58:50.580 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64126.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64126.log.gz 2026-03-23T19:58:50.581 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61397.log 2026-03-23T19:58:50.587 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64224.log 2026-03-23T19:58:50.588 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64175.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64175.log.gz 2026-03-23T19:58:50.589 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61446.log 2026-03-23T19:58:50.589 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61397.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61397.log.gz 2026-03-23T19:58:50.594 INFO:teuthology.orchestra.run.vm04.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-03-23T19:58:50.595 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64273.log 2026-03-23T19:58:50.595 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64224.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64224.log.gz 2026-03-23T19:58:50.602 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64322.log 2026-03-23T19:58:50.602 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64273.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64273.log.gz 2026-03-23T19:58:50.606 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64371.log 2026-03-23T19:58:50.606 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64322.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64322.log.gz 2026-03-23T19:58:50.606 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64420.log 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64371.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64371.log.gz 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64469.log 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64420.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64420.log.gz 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64518.log 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64567.log 2026-03-23T19:58:50.607 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64469.log.gz 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64518.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64518.log.gz 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64616.log 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61495.log 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.64567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64665.log 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64567.log.gz 2026-03-23T19:58:50.608 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61446.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61446.log.gz 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64714.log 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64616.log.gz 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64665.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64665.log.gz 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64763.log 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64812.log 2026-03-23T19:58:50.609 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64714.log.gz 2026-03-23T19:58:50.610 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65081.log 2026-03-23T19:58:50.614 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64763.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64763.log.gz 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65105.log 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.64812.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.64812.log.gz 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65129.log 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65081.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.65081.log.gz 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65146.log 2026-03-23T19:58:50.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65105.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.65105.log.gz 2026-03-23T19:58:50.616 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65163.log 2026-03-23T19:58:50.623 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61544.log 2026-03-23T19:58:50.623 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61495.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61495.log.gz 2026-03-23T19:58:50.629 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65146.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.65146.log.gz 2026-03-23T19:58:50.632 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65129.log: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.65129.log.gz 2026-03-23T19:58:50.633 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65180.log 2026-03-23T19:58:50.634 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65163.log: 70.1% -- replaced with /var/log/ceph/cluster1-client.admin.65163.log.gz 2026-03-23T19:58:50.638 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61593.log 2026-03-23T19:58:50.638 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65197.log 2026-03-23T19:58:50.638 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61544.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61544.log.gz 2026-03-23T19:58:50.646 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65214.log 2026-03-23T19:58:50.647 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65197.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65197.log.gz 2026-03-23T19:58:50.648 INFO:teuthology.orchestra.run.vm08.stderr: 61.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65180.log.gz 2026-03-23T19:58:50.652 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61642.log 2026-03-23T19:58:50.653 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61593.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61593.log.gz 2026-03-23T19:58:50.653 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65231.log 2026-03-23T19:58:50.658 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61691.log 2026-03-23T19:58:50.658 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61642.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61642.log.gz 2026-03-23T19:58:50.663 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65214.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65214.log.gz 2026-03-23T19:58:50.663 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61740.log 2026-03-23T19:58:50.663 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61691.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61691.log.gz 2026-03-23T19:58:50.666 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65248.log 2026-03-23T19:58:50.667 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65231.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65231.log.gz 2026-03-23T19:58:50.673 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65265.log 2026-03-23T19:58:50.677 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65248.log: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65248.log.gz 2026-03-23T19:58:50.678 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61789.log 2026-03-23T19:58:50.678 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61740.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61740.log.gz 2026-03-23T19:58:50.680 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65282.log 2026-03-23T19:58:50.690 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65265.log: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65265.log.gz 2026-03-23T19:58:50.691 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61838.log 2026-03-23T19:58:50.691 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61789.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61789.log.gz 2026-03-23T19:58:50.692 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61887.log 2026-03-23T19:58:50.692 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61838.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61838.log.gz 2026-03-23T19:58:50.695 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65299.log 2026-03-23T19:58:50.696 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65282.log: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65282.log.gz 2026-03-23T19:58:50.704 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61937.log 2026-03-23T19:58:50.704 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65331.log 2026-03-23T19:58:50.704 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65299.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.65299.log.gz 2026-03-23T19:58:50.704 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61887.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61887.log.gz 2026-03-23T19:58:50.710 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61984.log 2026-03-23T19:58:50.710 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61937.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61937.log.gz 2026-03-23T19:58:50.711 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65355.log 2026-03-23T19:58:50.712 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65331.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.65331.log.gz 2026-03-23T19:58:50.715 INFO:teuthology.orchestra.run.vm04.stderr:/var/log/ceph/cluster1-client.admin.61984.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.61984.log.gz 2026-03-23T19:58:50.718 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65372.log 2026-03-23T19:58:50.722 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65355.log: 56.5% -- replaced with /var/log/ceph/cluster1-client.admin.65355.log.gz 2026-03-23T19:58:50.727 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65389.log 2026-03-23T19:58:50.730 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65406.log 2026-03-23T19:58:50.731 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65423.log 2026-03-23T19:58:50.731 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65449.log 2026-03-23T19:58:50.734 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65389.log: 53.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65372.log.gz 2026-03-23T19:58:50.734 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65406.log: /var/log/ceph/cluster2-client.admin.65423.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65475.log 2026-03-23T19:58:50.737 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65449.log: 56.8% -- replaced with /var/log/ceph/cluster1-client.admin.65406.log.gz 2026-03-23T19:58:50.737 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65492.log 2026-03-23T19:58:50.738 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65475.log: 61.7% -- replaced with /var/log/ceph/cluster1-client.admin.65389.log.gz 2026-03-23T19:58:50.738 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.65423.log.gz 2026-03-23T19:58:50.738 INFO:teuthology.orchestra.run.vm08.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.65475.log.gz 2026-03-23T19:58:50.738 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65509.log 2026-03-23T19:58:50.739 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.65449.log.gz 2026-03-23T19:58:50.741 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65526.log 2026-03-23T19:58:50.745 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65509.log: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.65492.log.gz 2026-03-23T19:58:50.746 INFO:teuthology.orchestra.run.vm08.stderr: 69.9% -- replaced with /var/log/ceph/cluster2-client.admin.65509.log.gz 2026-03-23T19:58:50.746 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65543.log 2026-03-23T19:58:50.746 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65526.log: 61.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65526.log.gz 2026-03-23T19:58:50.751 INFO:teuthology.orchestra.run.vm04.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-03-23T19:58:50.752 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65560.log 2026-03-23T19:58:50.755 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65543.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65543.log.gz 2026-03-23T19:58:50.758 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65577.log 2026-03-23T19:58:50.758 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65594.log 2026-03-23T19:58:50.761 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65560.log: /var/log/ceph/cluster2-client.mirror.65577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65611.log 2026-03-23T19:58:50.763 INFO:teuthology.orchestra.run.vm08.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65577.log.gz 2026-03-23T19:58:50.763 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65594.log: 64.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65594.log.gz 2026-03-23T19:58:50.764 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65628.log 2026-03-23T19:58:50.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65645.log 2026-03-23T19:58:50.772 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65611.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65560.log.gz 2026-03-23T19:58:50.772 INFO:teuthology.orchestra.run.vm08.stderr: 64.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65611.log.gz 2026-03-23T19:58:50.774 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65628.log: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65628.log.gz 2026-03-23T19:58:50.780 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65677.log 2026-03-23T19:58:50.780 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65645.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.65645.log.gz 2026-03-23T19:58:50.788 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65701.log 2026-03-23T19:58:50.788 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.65677.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.65677.log.gz 2026-03-23T19:58:50.799 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65718.log 2026-03-23T19:58:50.800 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65701.log: 55.8% -- replaced with /var/log/ceph/cluster2-client.admin.65701.log.gz 2026-03-23T19:58:50.806 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65735.log 2026-03-23T19:58:50.811 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.65718.log: 53.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65718.log.gz 2026-03-23T19:58:50.815 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65752.log 2026-03-23T19:58:50.823 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.65773.log 2026-03-23T19:58:50.824 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.65752.log: 56.4% -- replaced with /var/log/ceph/cluster2-client.admin.65752.log.gz 2026-03-23T19:58:50.824 INFO:teuthology.orchestra.run.vm08.stderr: 60.7% -- replaced with /var/log/ceph/cluster2-client.admin.65735.log.gz 2026-03-23T19:58:50.830 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.65782.log 2026-03-23T19:58:50.837 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.65773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.65792.log 2026-03-23T19:58:50.839 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.65782.log: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.65773.log.gz 2026-03-23T19:58:50.848 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.65826.log 2026-03-23T19:58:50.849 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.65792.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.65792.log.gz 2026-03-23T19:58:50.855 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.65837.log 2026-03-23T19:58:50.864 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.65826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.65887.log 2026-03-23T19:58:50.867 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.65837.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.65826.log.gz 2026-03-23T19:58:50.868 INFO:teuthology.orchestra.run.vm08.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.65782.log.gz 2026-03-23T19:58:50.873 INFO:teuthology.orchestra.run.vm08.stderr: 18.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.65837.log.gz 2026-03-23T19:58:50.874 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.65895.log 2026-03-23T19:58:50.882 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.65887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.65957.log 2026-03-23T19:58:50.883 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.65895.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.65887.log.gz 2026-03-23T19:58:50.883 INFO:teuthology.orchestra.run.vm08.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.65895.log.gz 2026-03-23T19:58:50.887 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65971.log 2026-03-23T19:58:50.888 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.65957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66087.log 2026-03-23T19:58:50.893 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.65971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66162.log 2026-03-23T19:58:50.895 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66087.log: 54.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65971.log.gz 2026-03-23T19:58:50.896 INFO:teuthology.orchestra.run.vm08.stderr: 54.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66087.log.gz 2026-03-23T19:58:50.899 INFO:teuthology.orchestra.run.vm08.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.65957.log.gz 2026-03-23T19:58:50.903 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66182.log 2026-03-23T19:58:50.903 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66162.log: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66162.log.gz 2026-03-23T19:58:50.916 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66199.log 2026-03-23T19:58:50.920 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66216.log 2026-03-23T19:58:50.922 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66199.log: 68.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66199.log.gz 2026-03-23T19:58:50.926 INFO:teuthology.orchestra.run.vm08.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66182.log.gz 2026-03-23T19:58:50.927 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66251.log 2026-03-23T19:58:50.937 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66216.log: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66216.log.gz 2026-03-23T19:58:50.941 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66268.log 2026-03-23T19:58:50.941 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66251.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66251.log.gz 2026-03-23T19:58:50.951 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66288.log 2026-03-23T19:58:50.952 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66268.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66268.log.gz 2026-03-23T19:58:50.958 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66400.log 2026-03-23T19:58:50.967 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66422.log 2026-03-23T19:58:50.968 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66400.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66288.log.gz 2026-03-23T19:58:50.975 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66449.log 2026-03-23T19:58:50.978 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66422.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66422.log.gz 2026-03-23T19:58:50.978 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66400.log.gz 2026-03-23T19:58:50.983 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66473.log 2026-03-23T19:58:50.993 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66449.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66449.log.gz 2026-03-23T19:58:50.995 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66497.log 2026-03-23T19:58:50.995 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66473.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66473.log.gz 2026-03-23T19:58:51.009 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66521.log 2026-03-23T19:58:51.009 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66497.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66497.log.gz 2026-03-23T19:58:51.013 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66545.log 2026-03-23T19:58:51.014 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66572.log 2026-03-23T19:58:51.014 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66521.log.gz 2026-03-23T19:58:51.024 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66605.log 2026-03-23T19:58:51.024 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.66545.log: /var/log/ceph/cluster1-client.admin.66572.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.66545.log.gz 2026-03-23T19:58:51.031 INFO:teuthology.orchestra.run.vm08.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.66572.log.gz 2026-03-23T19:58:51.031 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66625.log 2026-03-23T19:58:51.040 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66605.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66648.log 2026-03-23T19:58:51.041 INFO:teuthology.orchestra.run.vm08.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66605.log.gz 2026-03-23T19:58:51.046 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66625.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66625.log.gz 2026-03-23T19:58:51.046 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66668.log 2026-03-23T19:58:51.056 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66648.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66648.log.gz 2026-03-23T19:58:51.056 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66691.log 2026-03-23T19:58:51.058 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66668.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66668.log.gz 2026-03-23T19:58:51.063 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66714.log 2026-03-23T19:58:51.065 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66691.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66691.log.gz 2026-03-23T19:58:51.069 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66735.log 2026-03-23T19:58:51.074 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66752.log 2026-03-23T19:58:51.074 INFO:teuthology.orchestra.run.vm08.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66714.log.gz 2026-03-23T19:58:51.079 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66772.log 2026-03-23T19:58:51.084 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66752.log: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66735.log.gz 2026-03-23T19:58:51.086 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66789.log 2026-03-23T19:58:51.088 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66772.log: 69.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66772.log.gz 2026-03-23T19:58:51.089 INFO:teuthology.orchestra.run.vm08.stderr: 54.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66752.log.gz 2026-03-23T19:58:51.099 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66806.log 2026-03-23T19:58:51.099 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66789.log: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66789.log.gz 2026-03-23T19:58:51.106 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66823.log 2026-03-23T19:58:51.115 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66843.log 2026-03-23T19:58:51.116 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66823.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66806.log.gz 2026-03-23T19:58:51.122 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66911.log 2026-03-23T19:58:51.123 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66843.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66823.log.gz 2026-03-23T19:58:51.124 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66843.log.gz 2026-03-23T19:58:51.131 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66933.log 2026-03-23T19:58:51.138 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66911.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66959.log 2026-03-23T19:58:51.142 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66933.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66911.log.gz 2026-03-23T19:58:51.147 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66983.log 2026-03-23T19:58:51.148 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.66959.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66959.log.gz 2026-03-23T19:58:51.148 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66933.log.gz 2026-03-23T19:58:51.151 INFO:teuthology.orchestra.run.vm08.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-03-23T19:58:51.152 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67007.log 2026-03-23T19:58:51.152 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.66983.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.66983.log.gz 2026-03-23T19:58:51.153 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67034.log 2026-03-23T19:58:51.153 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.67007.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.67007.log.gz 2026-03-23T19:58:51.153 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67068.log 2026-03-23T19:58:51.154 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.67034.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67034.log.gz 2026-03-23T19:58:51.154 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67088.log 2026-03-23T19:58:51.156 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67068.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67068.log.gz 2026-03-23T19:58:51.156 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67119.log 2026-03-23T19:58:51.158 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67088.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67088.log.gz 2026-03-23T19:58:51.158 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67139.log 2026-03-23T19:58:51.159 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67119.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67119.log.gz 2026-03-23T19:58:51.159 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67164.log 2026-03-23T19:58:51.161 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67139.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67139.log.gz 2026-03-23T19:58:51.161 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67187.log 2026-03-23T19:58:51.161 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67164.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67164.log.gz 2026-03-23T19:58:51.162 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67208.log 2026-03-23T19:58:51.162 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67187.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67187.log.gz 2026-03-23T19:58:51.163 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67225.log 2026-03-23T19:58:51.163 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67208.log: 54.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67208.log.gz 2026-03-23T19:58:51.163 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67242.log 2026-03-23T19:58:51.164 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67225.log: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67225.log.gz 2026-03-23T19:58:51.164 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67259.log 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67242.log: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67242.log.gz 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67276.log 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67259.log: 68.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67259.log.gz 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67293.log 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67276.log: 64.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67276.log.gz 2026-03-23T19:58:51.166 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67313.log 2026-03-23T19:58:51.167 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67293.log: 64.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67293.log.gz 2026-03-23T19:58:51.167 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67330.log 2026-03-23T19:58:51.167 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67313.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67313.log.gz 2026-03-23T19:58:51.167 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67350.log 2026-03-23T19:58:51.168 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67330.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67330.log.gz 2026-03-23T19:58:51.168 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67370.log 2026-03-23T19:58:51.169 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67390.log 2026-03-23T19:58:51.169 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67350.log.gz 2026-03-23T19:58:51.170 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67370.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67370.log.gz 2026-03-23T19:58:51.170 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67458.log 2026-03-23T19:58:51.171 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67390.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67390.log.gz 2026-03-23T19:58:51.171 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67512.log 2026-03-23T19:58:51.175 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67458.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67458.log.gz 2026-03-23T19:58:51.176 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67538.log 2026-03-23T19:58:51.177 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67512.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67512.log.gz 2026-03-23T19:58:51.177 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67562.log 2026-03-23T19:58:51.178 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67538.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67538.log.gz 2026-03-23T19:58:51.178 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67586.log 2026-03-23T19:58:51.178 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67562.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67562.log.gz 2026-03-23T19:58:51.179 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67611.log 2026-03-23T19:58:51.179 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67586.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67586.log.gz 2026-03-23T19:58:51.180 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67635.log 2026-03-23T19:58:51.180 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67611.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67611.log.gz 2026-03-23T19:58:51.180 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67662.log 2026-03-23T19:58:51.181 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.67635.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.67635.log.gz 2026-03-23T19:58:51.181 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67689.log 2026-03-23T19:58:51.182 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.67662.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.67662.log.gz 2026-03-23T19:58:51.182 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67716.log 2026-03-23T19:58:51.183 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.67689.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67689.log.gz 2026-03-23T19:58:51.183 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67749.log 2026-03-23T19:58:51.184 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.67716.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.67716.log.gz 2026-03-23T19:58:51.184 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67769.log 2026-03-23T19:58:51.184 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67749.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67791.log 2026-03-23T19:58:51.185 INFO:teuthology.orchestra.run.vm08.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67749.log.gz 2026-03-23T19:58:51.187 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67769.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67769.log.gz 2026-03-23T19:58:51.187 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67811.log 2026-03-23T19:58:51.187 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67791.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.67791.log.gz 2026-03-23T19:58:51.188 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67935.log 2026-03-23T19:58:51.188 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.67811.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.67811.log.gz 2026-03-23T19:58:51.188 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67952.log 2026-03-23T19:58:51.189 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67935.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67935.log.gz 2026-03-23T19:58:51.189 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67972.log 2026-03-23T19:58:51.190 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67952.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67952.log.gz 2026-03-23T19:58:51.190 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67992.log 2026-03-23T19:58:51.191 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67972.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67972.log.gz 2026-03-23T19:58:51.191 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.68014.log 2026-03-23T19:58:51.191 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.67992.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.68023.log 2026-03-23T19:58:51.195 INFO:teuthology.orchestra.run.vm08.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67992.log.gz 2026-03-23T19:58:51.195 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.68014.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.68014.log.gz 2026-03-23T19:58:51.195 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.68028.log 2026-03-23T19:58:51.209 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.68023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.68067.log 2026-03-23T19:58:51.209 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.68028.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.68028.log.gz 2026-03-23T19:58:51.210 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.68072.log 2026-03-23T19:58:51.228 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.68067.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.68119.log 2026-03-23T19:58:51.229 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.68072.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.68072.log.gz 2026-03-23T19:58:51.233 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.68121.log 2026-03-23T19:58:51.234 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.68205.log 2026-03-23T19:58:51.234 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68440.log 2026-03-23T19:58:51.235 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68466.log 2026-03-23T19:58:51.246 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.68119.log: /var/log/ceph/cluster1-client.mirror.3.68205.log: /var/log/ceph/cluster1-client.mirror.3.68121.log: 18.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.68121.log.gz 2026-03-23T19:58:51.246 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68490.log 2026-03-23T19:58:51.249 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.68466.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68466.log.gz 2026-03-23T19:58:51.250 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68514.log 2026-03-23T19:58:51.250 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68490.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68490.log.gz 2026-03-23T19:58:51.250 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68546.log 2026-03-23T19:58:51.251 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68514.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68514.log.gz 2026-03-23T19:58:51.251 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68570.log 2026-03-23T19:58:51.254 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68546.log: /var/log/ceph/cluster2-client.mirror.68440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68597.log 2026-03-23T19:58:51.254 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68546.log.gz 2026-03-23T19:58:51.257 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.68570.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68570.log.gz 2026-03-23T19:58:51.259 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68440.log.gz 2026-03-23T19:58:51.262 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68624.log 2026-03-23T19:58:51.262 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.68597.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.68597.log.gz 2026-03-23T19:58:51.262 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68651.log 2026-03-23T19:58:51.263 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.68624.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.68624.log.gz 2026-03-23T19:58:51.263 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68684.log 2026-03-23T19:58:51.264 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.68651.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.68651.log.gz 2026-03-23T19:58:51.264 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68704.log 2026-03-23T19:58:51.267 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68729.log 2026-03-23T19:58:51.290 INFO:teuthology.orchestra.run.vm04.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-03-23T19:58:51.290 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68769.log 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.68684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68795.log 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68684.log.gz 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.68769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68819.log 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68704.log.gz 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68769.log.gz 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.68795.log: /var/log/ceph/cluster2-client.mirror.68729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68843.log 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68795.log.gz 2026-03-23T19:58:51.291 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68819.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68819.log.gz 2026-03-23T19:58:51.299 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68870.log 2026-03-23T19:58:51.300 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68903.log 2026-03-23T19:58:51.306 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.68870.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68729.log.gz 2026-03-23T19:58:51.306 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.68843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68923.log 2026-03-23T19:58:51.307 INFO:teuthology.orchestra.run.vm08.stderr: 94.2%/var/log/ceph/cluster2-client.mirror.68903.log: -- replaced with /var/log/ceph/cluster1-client.mirror.3.68205.log.gz 2026-03-23T19:58:51.307 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68843.log.gz 2026-03-23T19:58:51.308 INFO:teuthology.orchestra.run.vm08.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68903.log.gz 2026-03-23T19:58:51.309 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68994.log 2026-03-23T19:58:51.309 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.68923.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.68870.log.gz 2026-03-23T19:58:51.310 INFO:teuthology.orchestra.run.vm08.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68923.log.gz 2026-03-23T19:58:51.310 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69021.log 2026-03-23T19:58:51.319 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.68119.log.gz 2026-03-23T19:58:51.320 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69048.log 2026-03-23T19:58:51.320 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.68994.log: /var/log/ceph/cluster1-client.admin.69021.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.69021.log.gz 2026-03-23T19:58:51.321 INFO:teuthology.orchestra.run.vm08.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.68067.log.gz 2026-03-23T19:58:51.321 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69075.log 2026-03-23T19:58:51.322 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69048.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69048.log.gz 2026-03-23T19:58:51.323 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69147.log 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69174.log 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69075.log: /var/log/ceph/cluster1-client.admin.69147.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69147.log.gz 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69201.log 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.69075.log.gz 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69228.log 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.admin.69201.log.gz 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69300.log 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69300.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69300.log.gz 2026-03-23T19:58:51.341 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69327.log 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69354.log 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69381.log 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69354.log: /var/log/ceph/cluster1-client.admin.69228.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68994.log.gz 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69453.log 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69480.log 2026-03-23T19:58:51.342 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69507.log 2026-03-23T19:58:51.343 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69534.log 2026-03-23T19:58:51.347 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.69327.log.gz 2026-03-23T19:58:51.347 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69453.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69354.log.gz 2026-03-23T19:58:51.348 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69480.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.69228.log.gz 2026-03-23T19:58:51.348 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69453.log.gz 2026-03-23T19:58:51.348 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.69381.log.gz 2026-03-23T19:58:51.348 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69507.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.69480.log.gz 2026-03-23T19:58:51.349 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69507.log.gz 2026-03-23T19:58:51.349 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69606.log 2026-03-23T19:58:51.349 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69633.log 2026-03-23T19:58:51.349 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69534.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.69174.log.gz 2026-03-23T19:58:51.350 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69606.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69606.log.gz 2026-03-23T19:58:51.354 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.69534.log.gz 2026-03-23T19:58:51.354 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69660.log 2026-03-23T19:58:51.359 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69633.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.69633.log.gz 2026-03-23T19:58:51.365 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69687.log 2026-03-23T19:58:51.368 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69660.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69660.log.gz 2026-03-23T19:58:51.369 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69759.log 2026-03-23T19:58:51.370 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69786.log 2026-03-23T19:58:51.371 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69759.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.69687.log.gz 2026-03-23T19:58:51.374 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69759.log.gz 2026-03-23T19:58:51.375 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69813.log 2026-03-23T19:58:51.378 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69840.log 2026-03-23T19:58:51.384 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69786.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.69786.log.gz 2026-03-23T19:58:51.394 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69813.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69813.log.gz 2026-03-23T19:58:51.394 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69893.log 2026-03-23T19:58:51.397 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69840.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.69840.log.gz 2026-03-23T19:58:51.403 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69920.log 2026-03-23T19:58:51.404 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69893.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69893.log.gz 2026-03-23T19:58:51.407 INFO:teuthology.orchestra.run.vm08.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.0.68023.log.gz 2026-03-23T19:58:51.408 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69976.log 2026-03-23T19:58:51.409 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69920.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.69920.log.gz 2026-03-23T19:58:51.409 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70003.log 2026-03-23T19:58:51.409 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.69976.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69976.log.gz 2026-03-23T19:58:51.409 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70056.log 2026-03-23T19:58:51.410 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70003.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.70003.log.gz 2026-03-23T19:58:51.410 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70083.log 2026-03-23T19:58:51.411 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70056.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70056.log.gz 2026-03-23T19:58:51.411 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70200.log 2026-03-23T19:58:51.411 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70083.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.70083.log.gz 2026-03-23T19:58:51.412 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70227.log 2026-03-23T19:58:51.412 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70200.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70254.log 2026-03-23T19:58:51.412 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70227.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70200.log.gz 2026-03-23T19:58:51.412 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70281.log 2026-03-23T19:58:51.413 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70254.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70254.log.gz 2026-03-23T19:58:51.413 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70353.log 2026-03-23T19:58:51.413 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.70227.log.gz 2026-03-23T19:58:51.414 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70281.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.70281.log.gz 2026-03-23T19:58:51.414 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70380.log 2026-03-23T19:58:51.414 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70353.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70407.log 2026-03-23T19:58:51.414 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70380.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70353.log.gz 2026-03-23T19:58:51.415 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70434.log 2026-03-23T19:58:51.415 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70484.log 2026-03-23T19:58:51.415 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70504.log 2026-03-23T19:58:51.416 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70528.log 2026-03-23T19:58:51.416 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70548.log 2026-03-23T19:58:51.416 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70528.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.70407.log.gz 2026-03-23T19:58:51.416 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.70434.log.gz 2026-03-23T19:58:51.417 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70586.log 2026-03-23T19:58:51.417 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70613.log 2026-03-23T19:58:51.419 INFO:teuthology.orchestra.run.vm08.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70504.log.gz 2026-03-23T19:58:51.443 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70548.log.gz 2026-03-23T19:58:51.443 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70676.log 2026-03-23T19:58:51.443 INFO:teuthology.orchestra.run.vm08.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70484.log.gz 2026-03-23T19:58:51.443 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70613.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70613.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70703.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70676.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.70676.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70729.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70703.log: 90.4% 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70703.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70749.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.70380.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70729.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70729.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70789.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70810.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70789.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70789.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70832.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70810.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70810.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70749.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70855.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.70832.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70832.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70876.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.70855.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70855.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70893.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70876.log: 65.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70876.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70913.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.70586.log: /var/log/ceph/cluster2-client.mirror.70893.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70528.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70893.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70967.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70913.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70913.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70987.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70967.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70967.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71012.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.70987.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.70586.log.gz 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70987.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71037.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71012.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71062.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71012.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.71037.log: --verbose -- /var/log/ceph/cluster1-client.mirror.71087.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71037.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71112.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71062.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71062.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71087.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71087.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71137.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71162.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71112.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71137.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71137.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71187.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71162.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71162.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71208.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71187.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71187.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71225.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71208.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71208.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71245.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71225.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71225.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71295.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71245.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71245.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71316.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71342.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71366.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71316.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71427.log 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71342.log.gz 2026-03-23T19:58:51.444 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71295.log.gz 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71366.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71366.log.gz 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71451.log 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71427.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71427.log.gz 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71475.log 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.71451.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71451.log.gz 2026-03-23T19:58:51.445 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71502.log 2026-03-23T19:58:51.446 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.71475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.71532.log 2026-03-23T19:58:51.446 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.71502.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.71475.log.gz 2026-03-23T19:58:51.447 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.71502.log.gz 2026-03-23T19:58:51.447 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.71541.log 2026-03-23T19:58:51.447 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.0.71532.log: 18.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.71532.log.gz 2026-03-23T19:58:51.447 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.71549.log 2026-03-23T19:58:51.449 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.0.71541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.71585.log 2026-03-23T19:58:51.449 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.1.71549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.71593.log 2026-03-23T19:58:51.450 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.1.71585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.71646.log 2026-03-23T19:58:51.460 INFO:teuthology.orchestra.run.vm08.stderr: 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.71549.log.gz 2026-03-23T19:58:51.474 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.2.71593.log: 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.71593.log.gz 2026-03-23T19:58:51.486 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.71654.log 2026-03-23T19:58:51.496 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.2.71646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.71730.log 2026-03-23T19:58:51.496 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.3.71654.log: 18.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.71654.log.gz 2026-03-23T19:58:51.506 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71738.log 2026-03-23T19:58:51.514 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.3.71730.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.71585.log.gz 2026-03-23T19:58:51.514 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71941.log 2026-03-23T19:58:51.518 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.71738.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71738.log.gz 2026-03-23T19:58:51.520 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71968.log 2026-03-23T19:58:51.527 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.71941.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.71941.log.gz 2026-03-23T19:58:51.527 INFO:teuthology.orchestra.run.vm08.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.71646.log.gz 2026-03-23T19:58:51.537 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71995.log 2026-03-23T19:58:51.537 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.71968.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.71730.log.gz 2026-03-23T19:58:51.538 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71968.log.gz 2026-03-23T19:58:51.548 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72021.log 2026-03-23T19:58:51.548 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.71995.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.71995.log.gz 2026-03-23T19:58:51.564 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72048.log 2026-03-23T19:58:51.564 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72021.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72021.log.gz 2026-03-23T19:58:51.578 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72074.log 2026-03-23T19:58:51.578 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72048.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72048.log.gz 2026-03-23T19:58:51.586 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72118.log 2026-03-23T19:58:51.589 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72074.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72074.log.gz 2026-03-23T19:58:51.592 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72139.log 2026-03-23T19:58:51.596 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72118.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72118.log.gz 2026-03-23T19:58:51.599 INFO:teuthology.orchestra.run.vm08.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.71541.log.gz 2026-03-23T19:58:51.599 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72165.log 2026-03-23T19:58:51.600 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72139.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72139.log.gz 2026-03-23T19:58:51.600 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72189.log 2026-03-23T19:58:51.600 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72165.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72165.log.gz 2026-03-23T19:58:51.601 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72213.log 2026-03-23T19:58:51.601 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72189.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72189.log.gz 2026-03-23T19:58:51.601 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72240.log 2026-03-23T19:58:51.602 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72213.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.72213.log.gz 2026-03-23T19:58:51.602 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72267.log 2026-03-23T19:58:51.602 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72240.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72240.log.gz 2026-03-23T19:58:51.603 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72294.log 2026-03-23T19:58:51.603 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72267.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.72267.log.gz 2026-03-23T19:58:51.603 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72327.log 2026-03-23T19:58:51.604 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72294.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72294.log.gz 2026-03-23T19:58:51.604 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72347.log 2026-03-23T19:58:51.605 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72327.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72327.log.gz 2026-03-23T19:58:51.605 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72369.log 2026-03-23T19:58:51.607 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72347.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.72347.log.gz 2026-03-23T19:58:51.607 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72403.log 2026-03-23T19:58:51.608 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72369.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72369.log.gz 2026-03-23T19:58:51.608 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72430.log 2026-03-23T19:58:51.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72403.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72403.log.gz 2026-03-23T19:58:51.609 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72457.log 2026-03-23T19:58:51.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72430.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72430.log.gz 2026-03-23T19:58:51.610 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72484.log 2026-03-23T19:58:51.610 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72457.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72457.log.gz 2026-03-23T19:58:51.610 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72510.log 2026-03-23T19:58:51.611 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72484.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72484.log.gz 2026-03-23T19:58:51.611 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72574.log 2026-03-23T19:58:51.612 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72510.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72510.log.gz 2026-03-23T19:58:51.612 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72607.log 2026-03-23T19:58:51.613 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72574.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72574.log.gz 2026-03-23T19:58:51.613 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72634.log 2026-03-23T19:58:51.614 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72607.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72607.log.gz 2026-03-23T19:58:51.614 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72661.log 2026-03-23T19:58:51.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72634.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72634.log.gz 2026-03-23T19:58:51.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72688.log 2026-03-23T19:58:51.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72661.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.72661.log.gz 2026-03-23T19:58:51.616 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72714.log 2026-03-23T19:58:51.616 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72688.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72688.log.gz 2026-03-23T19:58:51.616 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72758.log 2026-03-23T19:58:51.617 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72714.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72714.log.gz 2026-03-23T19:58:51.617 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72784.log 2026-03-23T19:58:51.618 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72758.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72758.log.gz 2026-03-23T19:58:51.618 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72808.log 2026-03-23T19:58:51.619 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72784.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72784.log.gz 2026-03-23T19:58:51.619 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72832.log 2026-03-23T19:58:51.619 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72808.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72808.log.gz 2026-03-23T19:58:51.620 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72859.log 2026-03-23T19:58:51.620 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72832.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.72832.log.gz 2026-03-23T19:58:51.620 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72886.log 2026-03-23T19:58:51.621 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.72859.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72859.log.gz 2026-03-23T19:58:51.621 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72913.log 2026-03-23T19:58:51.622 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72886.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.72886.log.gz 2026-03-23T19:58:51.622 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72946.log 2026-03-23T19:58:51.622 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.72913.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72913.log.gz 2026-03-23T19:58:51.623 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72966.log 2026-03-23T19:58:51.623 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72988.log 2026-03-23T19:58:51.624 INFO:teuthology.orchestra.run.vm08.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72946.log.gz 2026-03-23T19:58:51.626 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.72966.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.72966.log.gz 2026-03-23T19:58:51.626 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73021.log 2026-03-23T19:58:51.627 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.72988.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72988.log.gz 2026-03-23T19:58:51.627 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73048.log 2026-03-23T19:58:51.628 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73021.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.73021.log.gz 2026-03-23T19:58:51.628 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73075.log 2026-03-23T19:58:51.628 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73048.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.73048.log.gz 2026-03-23T19:58:51.629 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73102.log 2026-03-23T19:58:51.629 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73075.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.73075.log.gz 2026-03-23T19:58:51.629 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73128.log 2026-03-23T19:58:51.630 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73102.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.73102.log.gz 2026-03-23T19:58:51.630 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73172.log 2026-03-23T19:58:51.631 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73128.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73128.log.gz 2026-03-23T19:58:51.631 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73193.log 2026-03-23T19:58:51.635 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73172.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73172.log.gz 2026-03-23T19:58:51.635 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73219.log 2026-03-23T19:58:51.636 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73193.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73193.log.gz 2026-03-23T19:58:51.636 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73243.log 2026-03-23T19:58:51.636 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73219.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73219.log.gz 2026-03-23T19:58:51.637 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73267.log 2026-03-23T19:58:51.637 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73243.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73243.log.gz 2026-03-23T19:58:51.637 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73294.log 2026-03-23T19:58:51.638 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73267.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.73267.log.gz 2026-03-23T19:58:51.638 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73321.log 2026-03-23T19:58:51.639 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73294.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.73294.log.gz 2026-03-23T19:58:51.639 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73348.log 2026-03-23T19:58:51.640 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73321.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.73321.log.gz 2026-03-23T19:58:51.640 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73381.log 2026-03-23T19:58:51.640 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73348.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.73348.log.gz 2026-03-23T19:58:51.641 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73401.log 2026-03-23T19:58:51.641 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73423.log 2026-03-23T19:58:51.642 INFO:teuthology.orchestra.run.vm08.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73381.log.gz 2026-03-23T19:58:51.643 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73401.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73401.log.gz 2026-03-23T19:58:51.644 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73468.log 2026-03-23T19:58:51.644 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73423.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.73423.log.gz 2026-03-23T19:58:51.645 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73495.log 2026-03-23T19:58:51.645 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73468.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73468.log.gz 2026-03-23T19:58:51.645 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73522.log 2026-03-23T19:58:51.646 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73495.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.73495.log.gz 2026-03-23T19:58:51.646 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73549.log 2026-03-23T19:58:51.647 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73522.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.73522.log.gz 2026-03-23T19:58:51.647 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73575.log 2026-03-23T19:58:51.647 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73549.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.73549.log.gz 2026-03-23T19:58:51.648 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73619.log 2026-03-23T19:58:51.648 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73641.log 2026-03-23T19:58:51.648 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73667.log 2026-03-23T19:58:51.649 INFO:teuthology.orchestra.run.vm08.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73575.log.gz 2026-03-23T19:58:51.652 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73619.log.gz 2026-03-23T19:58:51.653 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73641.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73641.log.gz 2026-03-23T19:58:51.654 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73691.log 2026-03-23T19:58:51.654 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73667.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73667.log.gz 2026-03-23T19:58:51.654 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73715.log 2026-03-23T19:58:51.655 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73691.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73691.log.gz 2026-03-23T19:58:51.655 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73742.log 2026-03-23T19:58:51.655 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73715.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.73715.log.gz 2026-03-23T19:58:51.656 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73769.log 2026-03-23T19:58:51.656 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73742.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.73742.log.gz 2026-03-23T19:58:51.656 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73796.log 2026-03-23T19:58:51.657 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73769.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.73769.log.gz 2026-03-23T19:58:51.657 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73829.log 2026-03-23T19:58:51.658 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73796.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.73796.log.gz 2026-03-23T19:58:51.658 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73849.log 2026-03-23T19:58:51.659 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73829.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73829.log.gz 2026-03-23T19:58:51.659 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73875.log 2026-03-23T19:58:51.660 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.73849.log: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73849.log.gz 2026-03-23T19:58:51.661 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73920.log 2026-03-23T19:58:51.661 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.73875.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73875.log.gz 2026-03-23T19:58:51.662 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73947.log 2026-03-23T19:58:51.662 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73920.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.73920.log.gz 2026-03-23T19:58:51.662 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73974.log 2026-03-23T19:58:51.663 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.73947.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73947.log.gz 2026-03-23T19:58:51.663 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74001.log 2026-03-23T19:58:51.664 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.73974.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.73974.log.gz 2026-03-23T19:58:51.664 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74027.log 2026-03-23T19:58:51.664 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74001.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.74001.log.gz 2026-03-23T19:58:51.665 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74059.log 2026-03-23T19:58:51.665 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74027.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.74027.log.gz 2026-03-23T19:58:51.666 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74085.log 2026-03-23T19:58:51.667 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74059.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74059.log.gz 2026-03-23T19:58:51.667 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74109.log 2026-03-23T19:58:51.667 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74085.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74085.log.gz 2026-03-23T19:58:51.668 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74133.log 2026-03-23T19:58:51.668 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74109.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74109.log.gz 2026-03-23T19:58:51.668 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74160.log 2026-03-23T19:58:51.669 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74187.log 2026-03-23T19:58:51.669 INFO:teuthology.orchestra.run.vm08.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.74133.log.gz 2026-03-23T19:58:51.669 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74214.log 2026-03-23T19:58:51.670 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.74160.log.gz 2026-03-23T19:58:51.670 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74187.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.74187.log.gz 2026-03-23T19:58:51.671 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74240.log 2026-03-23T19:58:51.671 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74285.log 2026-03-23T19:58:51.671 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.74214.log.gz 2026-03-23T19:58:51.672 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74240.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74240.log.gz 2026-03-23T19:58:51.672 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74312.log 2026-03-23T19:58:51.673 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74285.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.74285.log.gz 2026-03-23T19:58:51.673 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74339.log 2026-03-23T19:58:51.673 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74312.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.74312.log.gz 2026-03-23T19:58:51.674 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74366.log 2026-03-23T19:58:51.674 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74339.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.74339.log.gz 2026-03-23T19:58:51.674 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74392.log 2026-03-23T19:58:51.675 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74366.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.74366.log.gz 2026-03-23T19:58:51.675 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74436.log 2026-03-23T19:58:51.676 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74392.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74392.log.gz 2026-03-23T19:58:51.676 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74462.log 2026-03-23T19:58:51.677 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74436.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74436.log.gz 2026-03-23T19:58:51.677 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74486.log 2026-03-23T19:58:51.678 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74462.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74462.log.gz 2026-03-23T19:58:51.678 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74510.log 2026-03-23T19:58:51.678 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74486.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74486.log.gz 2026-03-23T19:58:51.679 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74537.log 2026-03-23T19:58:51.679 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74510.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.74510.log.gz 2026-03-23T19:58:51.679 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74564.log 2026-03-23T19:58:51.680 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74537.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.74537.log.gz 2026-03-23T19:58:51.681 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74591.log 2026-03-23T19:58:51.681 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74564.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.74564.log.gz 2026-03-23T19:58:51.681 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74617.log 2026-03-23T19:58:51.682 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74591.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.74591.log.gz 2026-03-23T19:58:51.682 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74662.log 2026-03-23T19:58:51.683 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74617.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74617.log.gz 2026-03-23T19:58:51.683 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74689.log 2026-03-23T19:58:51.684 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74662.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.74662.log.gz 2026-03-23T19:58:51.684 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74716.log 2026-03-23T19:58:51.684 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74689.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74689.log.gz 2026-03-23T19:58:51.685 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74743.log 2026-03-23T19:58:51.685 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74716.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.74716.log.gz 2026-03-23T19:58:51.685 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74769.log 2026-03-23T19:58:51.686 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74743.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.74743.log.gz 2026-03-23T19:58:51.686 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74801.log 2026-03-23T19:58:51.687 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74769.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74769.log.gz 2026-03-23T19:58:51.687 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74827.log 2026-03-23T19:58:51.688 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74801.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74801.log.gz 2026-03-23T19:58:51.689 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74851.log 2026-03-23T19:58:51.689 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74827.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74827.log.gz 2026-03-23T19:58:51.689 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74875.log 2026-03-23T19:58:51.690 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.74851.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74851.log.gz 2026-03-23T19:58:51.690 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74902.log 2026-03-23T19:58:51.691 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74929.log 2026-03-23T19:58:51.691 INFO:teuthology.orchestra.run.vm08.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.74875.log.gz 2026-03-23T19:58:51.691 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.74902.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.74902.log.gz 2026-03-23T19:58:51.692 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74956.log 2026-03-23T19:58:51.692 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74929.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.74929.log.gz 2026-03-23T19:58:51.692 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74982.log 2026-03-23T19:58:51.693 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.74956.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.74956.log.gz 2026-03-23T19:58:51.693 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75027.log 2026-03-23T19:58:51.694 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.74982.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74982.log.gz 2026-03-23T19:58:51.694 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75054.log 2026-03-23T19:58:51.695 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75027.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.75027.log.gz 2026-03-23T19:58:51.695 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75081.log 2026-03-23T19:58:51.695 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75108.log 2026-03-23T19:58:51.695 INFO:teuthology.orchestra.run.vm08.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.75054.log.gz 2026-03-23T19:58:51.696 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75081.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.75081.log.gz 2026-03-23T19:58:51.696 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75134.log 2026-03-23T19:58:51.697 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75108.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.75108.log.gz 2026-03-23T19:58:51.697 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75178.log 2026-03-23T19:58:51.698 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75134.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75134.log.gz 2026-03-23T19:58:51.698 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75204.log 2026-03-23T19:58:51.699 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75178.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75178.log.gz 2026-03-23T19:58:51.699 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75228.log 2026-03-23T19:58:51.700 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75204.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75204.log.gz 2026-03-23T19:58:51.700 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75252.log 2026-03-23T19:58:51.700 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75228.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75228.log.gz 2026-03-23T19:58:51.701 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75279.log 2026-03-23T19:58:51.701 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75252.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.75252.log.gz 2026-03-23T19:58:51.701 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75306.log 2026-03-23T19:58:51.702 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75279.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.75279.log.gz 2026-03-23T19:58:51.702 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75333.log 2026-03-23T19:58:51.702 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75306.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.75306.log.gz 2026-03-23T19:58:51.703 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75359.log 2026-03-23T19:58:51.703 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75333.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.75333.log.gz 2026-03-23T19:58:51.704 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75404.log 2026-03-23T19:58:51.704 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75359.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75359.log.gz 2026-03-23T19:58:51.704 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75431.log 2026-03-23T19:58:51.705 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75404.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.75404.log.gz 2026-03-23T19:58:51.706 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75458.log 2026-03-23T19:58:51.706 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75431.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.75431.log.gz 2026-03-23T19:58:51.707 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75485.log 2026-03-23T19:58:51.707 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75458.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.75458.log.gz 2026-03-23T19:58:51.707 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75511.log 2026-03-23T19:58:51.708 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75485.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.75485.log.gz 2026-03-23T19:58:51.708 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75543.log 2026-03-23T19:58:51.710 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75511.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.75511.log.gz 2026-03-23T19:58:51.710 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75569.log 2026-03-23T19:58:51.711 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75543.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75543.log.gz 2026-03-23T19:58:51.711 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75593.log 2026-03-23T19:58:51.712 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75569.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75569.log.gz 2026-03-23T19:58:51.712 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75617.log 2026-03-23T19:58:51.712 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75593.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75593.log.gz 2026-03-23T19:58:51.713 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75644.log 2026-03-23T19:58:51.713 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75617.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.75617.log.gz 2026-03-23T19:58:51.713 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75671.log 2026-03-23T19:58:51.714 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75644.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.75644.log.gz 2026-03-23T19:58:51.714 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75698.log 2026-03-23T19:58:51.715 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75671.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.75671.log.gz 2026-03-23T19:58:51.715 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75724.log 2026-03-23T19:58:51.715 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75698.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.75698.log.gz 2026-03-23T19:58:51.715 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75769.log 2026-03-23T19:58:51.716 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75724.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75724.log.gz 2026-03-23T19:58:51.716 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75796.log 2026-03-23T19:58:51.717 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75769.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.75769.log.gz 2026-03-23T19:58:51.717 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75823.log 2026-03-23T19:58:51.718 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.75796.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.75796.log.gz 2026-03-23T19:58:51.718 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75850.log 2026-03-23T19:58:51.718 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75823.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.75823.log.gz 2026-03-23T19:58:51.719 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75876.log 2026-03-23T19:58:51.719 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75850.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.75850.log.gz 2026-03-23T19:58:51.719 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75920.log 2026-03-23T19:58:51.720 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75876.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75876.log.gz 2026-03-23T19:58:51.721 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75946.log 2026-03-23T19:58:51.721 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75970.log 2026-03-23T19:58:51.722 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75920.log.gz 2026-03-23T19:58:51.722 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.75946.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75946.log.gz 2026-03-23T19:58:51.722 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75994.log 2026-03-23T19:58:51.723 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.75970.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75970.log.gz 2026-03-23T19:58:51.723 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76021.log 2026-03-23T19:58:51.724 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.75994.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.75994.log.gz 2026-03-23T19:58:51.724 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76050.log 2026-03-23T19:58:51.724 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76021.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.76021.log.gz 2026-03-23T19:58:51.725 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76077.log 2026-03-23T19:58:51.725 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76050.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76050.log.gz 2026-03-23T19:58:51.725 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76103.log 2026-03-23T19:58:51.726 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76077.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.76077.log.gz 2026-03-23T19:58:51.726 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76148.log 2026-03-23T19:58:51.727 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76103.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76103.log.gz 2026-03-23T19:58:51.728 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76175.log 2026-03-23T19:58:51.728 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76148.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76148.log.gz 2026-03-23T19:58:51.728 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76202.log 2026-03-23T19:58:51.729 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76175.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76175.log.gz 2026-03-23T19:58:51.729 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76229.log 2026-03-23T19:58:51.730 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76255.log 2026-03-23T19:58:51.730 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.76202.log.gz 2026-03-23T19:58:51.730 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76229.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.76229.log.gz 2026-03-23T19:58:51.731 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76311.log 2026-03-23T19:58:51.732 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.76255.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76255.log.gz 2026-03-23T19:58:51.732 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76337.log 2026-03-23T19:58:51.732 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.76311.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76361.log 2026-03-23T19:58:51.733 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.76311.log.gz 2026-03-23T19:58:51.733 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.76337.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.76337.log.gz 2026-03-23T19:58:51.734 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76385.log 2026-03-23T19:58:51.734 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76361.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76361.log.gz 2026-03-23T19:58:51.735 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76412.log 2026-03-23T19:58:51.735 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76385.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76439.log 2026-03-23T19:58:51.735 INFO:teuthology.orchestra.run.vm08.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.76385.log.gz 2026-03-23T19:58:51.736 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76412.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.76412.log.gz 2026-03-23T19:58:51.736 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76466.log 2026-03-23T19:58:51.737 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76439.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.76439.log.gz 2026-03-23T19:58:51.737 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76492.log 2026-03-23T19:58:51.737 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76466.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.76466.log.gz 2026-03-23T19:58:51.738 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76538.log 2026-03-23T19:58:51.738 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.76492.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76565.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76538.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76492.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76538.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76592.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76565.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.76565.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76619.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76592.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.76592.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76645.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76619.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.76619.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76677.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76645.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76645.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76703.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76677.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76677.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76727.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76703.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76703.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76751.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.76727.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.76727.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76778.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76751.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.76751.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76805.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76778.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.76778.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76832.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76805.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.76805.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76858.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76832.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.76832.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76903.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.76858.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76858.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76930.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76903.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76903.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76957.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.76930.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.76930.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76984.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76957.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.76957.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77010.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.76984.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.76984.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77042.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77010.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77010.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77068.log 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77042.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77042.log.gz 2026-03-23T19:58:51.766 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77092.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77068.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77068.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77116.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77092.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77092.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77143.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77116.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.77116.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77170.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77143.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.77143.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77197.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77170.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.77170.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77223.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77197.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.77197.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77268.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77223.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.77223.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77295.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77268.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.77268.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77322.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77295.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.77295.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77349.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77322.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.77322.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77375.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77419.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.77349.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77375.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77375.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77445.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77419.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77419.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77469.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77445.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77445.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77493.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77469.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77469.log.gz 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77520.log 2026-03-23T19:58:51.767 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77493.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.77493.log.gz 2026-03-23T19:58:51.768 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77547.log 2026-03-23T19:58:51.768 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77520.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77574.log 2026-03-23T19:58:51.768 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.77520.log.gz 2026-03-23T19:58:51.769 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77547.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.77547.log.gz 2026-03-23T19:58:51.769 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77600.log 2026-03-23T19:58:51.770 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77633.log 2026-03-23T19:58:51.770 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77660.log 2026-03-23T19:58:51.770 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77633.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.77574.log.gz 2026-03-23T19:58:51.771 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77600.log.gz 2026-03-23T19:58:51.771 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.77633.log.gz 2026-03-23T19:58:51.772 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77687.log 2026-03-23T19:58:51.772 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77714.log 2026-03-23T19:58:51.772 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.77660.log.gz 2026-03-23T19:58:51.773 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77740.log 2026-03-23T19:58:51.773 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.77687.log.gz 2026-03-23T19:58:51.774 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77714.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.77714.log.gz 2026-03-23T19:58:51.774 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77784.log 2026-03-23T19:58:51.774 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77810.log 2026-03-23T19:58:51.775 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77784.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77740.log.gz 2026-03-23T19:58:51.776 INFO:teuthology.orchestra.run.vm08.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.77784.log.gz 2026-03-23T19:58:51.776 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77834.log 2026-03-23T19:58:51.776 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77810.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77810.log.gz 2026-03-23T19:58:51.777 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77858.log 2026-03-23T19:58:51.777 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.77834.log: 86.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77834.log.gz 2026-03-23T19:58:51.777 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77885.log 2026-03-23T19:58:51.778 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77858.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.77858.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77912.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77939.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77912.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77965.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.77939.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.77912.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.77885.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.77939.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77998.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.77965.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.77965.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78025.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.77998.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.77998.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78052.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78025.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78025.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78079.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78052.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.78052.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78105.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78079.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.78079.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78149.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78105.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78105.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78175.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78149.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.78149.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78199.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78175.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78175.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78223.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78199.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78199.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78250.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78223.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.78223.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78277.log 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78250.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.78250.log.gz 2026-03-23T19:58:51.795 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78304.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78277.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.78277.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78330.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78304.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78304.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78376.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78330.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78330.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78403.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78430.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78376.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78403.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78457.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.78403.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78483.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78457.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.78430.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.78457.log.gz 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78515.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78541.log 2026-03-23T19:58:51.796 INFO:teuthology.orchestra.run.vm08.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78483.log.gz 2026-03-23T19:58:51.797 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78515.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78515.log.gz 2026-03-23T19:58:51.797 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78565.log 2026-03-23T19:58:51.797 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78541.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78541.log.gz 2026-03-23T19:58:51.798 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78589.log 2026-03-23T19:58:51.798 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78565.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78565.log.gz 2026-03-23T19:58:51.798 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78616.log 2026-03-23T19:58:51.799 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78589.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.78589.log.gz 2026-03-23T19:58:51.799 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78643.log 2026-03-23T19:58:51.800 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78616.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.78616.log.gz 2026-03-23T19:58:51.800 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78670.log 2026-03-23T19:58:51.800 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78643.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.78643.log.gz 2026-03-23T19:58:51.801 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78696.log 2026-03-23T19:58:51.801 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78670.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.78670.log.gz 2026-03-23T19:58:51.801 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78741.log 2026-03-23T19:58:51.802 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78696.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78696.log.gz 2026-03-23T19:58:51.802 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78768.log 2026-03-23T19:58:51.803 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78741.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.78741.log.gz 2026-03-23T19:58:51.803 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78795.log 2026-03-23T19:58:51.804 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.78768.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78768.log.gz 2026-03-23T19:58:51.804 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78822.log 2026-03-23T19:58:51.804 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78795.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.78795.log.gz 2026-03-23T19:58:51.805 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78848.log 2026-03-23T19:58:51.805 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78822.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.78822.log.gz 2026-03-23T19:58:51.805 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78880.log 2026-03-23T19:58:51.806 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78848.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78848.log.gz 2026-03-23T19:58:51.806 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78906.log 2026-03-23T19:58:51.807 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78930.log 2026-03-23T19:58:51.808 INFO:teuthology.orchestra.run.vm08.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78880.log.gz 2026-03-23T19:58:51.808 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.78906.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78906.log.gz 2026-03-23T19:58:51.809 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78954.log 2026-03-23T19:58:51.809 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.78930.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78930.log.gz 2026-03-23T19:58:51.809 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78981.log 2026-03-23T19:58:51.810 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78954.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.78954.log.gz 2026-03-23T19:58:51.810 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79008.log 2026-03-23T19:58:51.811 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.78981.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.78981.log.gz 2026-03-23T19:58:51.811 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79035.log 2026-03-23T19:58:51.811 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79008.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79008.log.gz 2026-03-23T19:58:51.811 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79061.log 2026-03-23T19:58:51.812 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79035.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79035.log.gz 2026-03-23T19:58:51.812 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79094.log 2026-03-23T19:58:51.813 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79061.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79061.log.gz 2026-03-23T19:58:51.813 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79121.log 2026-03-23T19:58:51.814 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79148.log 2026-03-23T19:58:51.814 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79094.log.gz 2026-03-23T19:58:51.814 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79175.log 2026-03-23T19:58:51.815 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79121.log.gz 2026-03-23T19:58:51.815 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79201.log 2026-03-23T19:58:51.816 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79175.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.79148.log.gz 2026-03-23T19:58:51.816 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.79175.log.gz 2026-03-23T19:58:51.816 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79233.log 2026-03-23T19:58:51.816 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79259.log 2026-03-23T19:58:51.817 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79201.log.gz 2026-03-23T19:58:51.818 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79233.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.79233.log.gz 2026-03-23T19:58:51.819 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79283.log 2026-03-23T19:58:51.819 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79259.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79259.log.gz 2026-03-23T19:58:51.819 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79307.log 2026-03-23T19:58:51.820 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79283.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79283.log.gz 2026-03-23T19:58:51.820 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79334.log 2026-03-23T19:58:51.821 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79307.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.79307.log.gz 2026-03-23T19:58:51.821 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79361.log 2026-03-23T19:58:51.821 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79334.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.79334.log.gz 2026-03-23T19:58:51.822 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79388.log 2026-03-23T19:58:51.822 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79361.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.79361.log.gz 2026-03-23T19:58:51.823 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79414.log 2026-03-23T19:58:51.823 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79388.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.79388.log.gz 2026-03-23T19:58:51.824 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79459.log 2026-03-23T19:58:51.825 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79414.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.79414.log.gz 2026-03-23T19:58:51.825 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79486.log 2026-03-23T19:58:51.826 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79459.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.79459.log.gz 2026-03-23T19:58:51.826 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79513.log 2026-03-23T19:58:51.827 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79486.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.79486.log.gz 2026-03-23T19:58:51.827 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79540.log 2026-03-23T19:58:51.828 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79513.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.79513.log.gz 2026-03-23T19:58:51.828 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79566.log 2026-03-23T19:58:51.828 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79540.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.79540.log.gz 2026-03-23T19:58:51.829 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79610.log 2026-03-23T19:58:51.830 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79566.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79566.log.gz 2026-03-23T19:58:51.830 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79636.log 2026-03-23T19:58:51.831 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79610.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79610.log.gz 2026-03-23T19:58:51.831 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79660.log 2026-03-23T19:58:51.832 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79636.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79636.log.gz 2026-03-23T19:58:51.832 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79684.log 2026-03-23T19:58:51.833 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79660.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79660.log.gz 2026-03-23T19:58:51.833 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79711.log 2026-03-23T19:58:51.834 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79684.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.79684.log.gz 2026-03-23T19:58:51.834 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79738.log 2026-03-23T19:58:51.834 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79765.log 2026-03-23T19:58:51.835 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.79711.log.gz 2026-03-23T19:58:51.835 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79738.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79738.log.gz 2026-03-23T19:58:51.836 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79791.log 2026-03-23T19:58:51.836 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79765.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.79765.log.gz 2026-03-23T19:58:51.836 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79836.log 2026-03-23T19:58:51.837 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.79791.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79791.log.gz 2026-03-23T19:58:51.838 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79863.log 2026-03-23T19:58:51.838 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79836.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.79836.log.gz 2026-03-23T19:58:51.839 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79890.log 2026-03-23T19:58:51.839 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.79863.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.79863.log.gz 2026-03-23T19:58:51.840 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79917.log 2026-03-23T19:58:51.840 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79890.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.79890.log.gz 2026-03-23T19:58:51.840 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79943.log 2026-03-23T19:58:51.841 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.79917.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.79917.log.gz 2026-03-23T19:58:51.841 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79999.log 2026-03-23T19:58:51.842 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79943.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79943.log.gz 2026-03-23T19:58:51.842 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80025.log 2026-03-23T19:58:51.843 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.79999.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.79999.log.gz 2026-03-23T19:58:51.844 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80049.log 2026-03-23T19:58:51.844 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80025.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.80025.log.gz 2026-03-23T19:58:51.844 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80073.log 2026-03-23T19:58:51.845 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80049.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80049.log.gz 2026-03-23T19:58:51.846 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80100.log 2026-03-23T19:58:51.846 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80073.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.80073.log.gz 2026-03-23T19:58:51.846 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80127.log 2026-03-23T19:58:51.847 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80100.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.80100.log.gz 2026-03-23T19:58:51.847 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80154.log 2026-03-23T19:58:51.847 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80127.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.80127.log.gz 2026-03-23T19:58:51.848 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80180.log 2026-03-23T19:58:51.848 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80154.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.80154.log.gz 2026-03-23T19:58:51.849 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80225.log 2026-03-23T19:58:51.849 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80180.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.80180.log.gz 2026-03-23T19:58:51.850 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80252.log 2026-03-23T19:58:51.850 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80225.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.80225.log.gz 2026-03-23T19:58:51.851 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80279.log 2026-03-23T19:58:51.851 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80252.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.80252.log.gz 2026-03-23T19:58:51.851 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80306.log 2026-03-23T19:58:51.852 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80279.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80279.log.gz 2026-03-23T19:58:51.852 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80332.log 2026-03-23T19:58:51.852 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80306.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.80306.log.gz 2026-03-23T19:58:51.853 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80364.log 2026-03-23T19:58:51.853 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80332.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80332.log.gz 2026-03-23T19:58:51.854 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80390.log 2026-03-23T19:58:51.855 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80364.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80364.log.gz 2026-03-23T19:58:51.855 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80414.log 2026-03-23T19:58:51.856 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80390.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80390.log.gz 2026-03-23T19:58:51.856 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80438.log 2026-03-23T19:58:51.856 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80414.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80465.log 2026-03-23T19:58:51.857 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80414.log.gz 2026-03-23T19:58:51.857 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80438.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80492.log 2026-03-23T19:58:51.857 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80465.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.80438.log.gz 2026-03-23T19:58:51.858 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80519.log 2026-03-23T19:58:51.858 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80545.log 2026-03-23T19:58:51.859 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80578.log 2026-03-23T19:58:51.859 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80605.log 2026-03-23T19:58:51.860 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.80465.log.gz 2026-03-23T19:58:51.860 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.80492.log.gz 2026-03-23T19:58:51.861 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80578.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.80519.log.gz 2026-03-23T19:58:51.861 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80545.log.gz 2026-03-23T19:58:51.861 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.80578.log.gz 2026-03-23T19:58:51.862 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80632.log 2026-03-23T19:58:51.862 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80605.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80659.log 2026-03-23T19:58:51.863 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.80605.log.gz 2026-03-23T19:58:51.863 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80632.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80632.log.gz 2026-03-23T19:58:51.863 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80685.log 2026-03-23T19:58:51.864 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80659.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.80659.log.gz 2026-03-23T19:58:51.864 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80717.log 2026-03-23T19:58:51.864 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80743.log 2026-03-23T19:58:51.865 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80717.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80767.log 2026-03-23T19:58:51.866 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80743.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80685.log.gz 2026-03-23T19:58:51.867 INFO:teuthology.orchestra.run.vm08.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80717.log.gz 2026-03-23T19:58:51.867 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80791.log 2026-03-23T19:58:51.867 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80743.log.gz 2026-03-23T19:58:51.868 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.80767.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80767.log.gz 2026-03-23T19:58:51.868 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80818.log 2026-03-23T19:58:51.869 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80791.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.80791.log.gz 2026-03-23T19:58:51.869 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80845.log 2026-03-23T19:58:51.869 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80818.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.80818.log.gz 2026-03-23T19:58:51.870 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80872.log 2026-03-23T19:58:51.870 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80845.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80845.log.gz 2026-03-23T19:58:51.870 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80898.log 2026-03-23T19:58:51.871 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80872.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.80872.log.gz 2026-03-23T19:58:51.871 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80943.log 2026-03-23T19:58:51.872 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.80898.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80898.log.gz 2026-03-23T19:58:51.872 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80970.log 2026-03-23T19:58:51.872 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80943.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.80943.log.gz 2026-03-23T19:58:51.873 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80997.log 2026-03-23T19:58:51.873 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.80970.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.80970.log.gz 2026-03-23T19:58:51.874 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81024.log 2026-03-23T19:58:51.874 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.80997.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.80997.log.gz 2026-03-23T19:58:51.874 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81050.log 2026-03-23T19:58:51.875 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81024.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.81024.log.gz 2026-03-23T19:58:51.875 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81094.log 2026-03-23T19:58:51.875 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81050.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81050.log.gz 2026-03-23T19:58:51.876 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81120.log 2026-03-23T19:58:51.877 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81094.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81094.log.gz 2026-03-23T19:58:51.877 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81144.log 2026-03-23T19:58:51.878 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81120.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81120.log.gz 2026-03-23T19:58:51.878 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81168.log 2026-03-23T19:58:51.878 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81195.log 2026-03-23T19:58:51.879 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81144.log.gz 2026-03-23T19:58:51.879 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81222.log 2026-03-23T19:58:51.879 INFO:teuthology.orchestra.run.vm08.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.81168.log.gz 2026-03-23T19:58:51.880 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81195.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.81195.log.gz 2026-03-23T19:58:51.880 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81249.log 2026-03-23T19:58:51.881 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81275.log 2026-03-23T19:58:51.881 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.81222.log.gz 2026-03-23T19:58:51.881 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81249.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.81249.log.gz 2026-03-23T19:58:51.882 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81320.log 2026-03-23T19:58:51.882 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81347.log 2026-03-23T19:58:51.883 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81275.log.gz 2026-03-23T19:58:51.883 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81374.log 2026-03-23T19:58:51.883 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.81320.log.gz 2026-03-23T19:58:51.884 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81347.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.81347.log.gz 2026-03-23T19:58:51.884 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81401.log 2026-03-23T19:58:51.885 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81374.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.81374.log.gz 2026-03-23T19:58:51.885 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81427.log 2026-03-23T19:58:51.886 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81483.log 2026-03-23T19:58:51.886 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.81401.log.gz 2026-03-23T19:58:51.886 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81509.log 2026-03-23T19:58:51.887 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81427.log.gz 2026-03-23T19:58:51.888 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81483.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.81483.log.gz 2026-03-23T19:58:51.888 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81533.log 2026-03-23T19:58:51.889 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81509.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81509.log.gz 2026-03-23T19:58:51.889 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81557.log 2026-03-23T19:58:51.889 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81533.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81533.log.gz 2026-03-23T19:58:51.889 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81584.log 2026-03-23T19:58:51.890 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81611.log 2026-03-23T19:58:51.890 INFO:teuthology.orchestra.run.vm08.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.81557.log.gz 2026-03-23T19:58:51.891 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81584.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.81584.log.gz 2026-03-23T19:58:51.891 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81638.log 2026-03-23T19:58:51.891 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81611.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.81611.log.gz 2026-03-23T19:58:51.892 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81664.log 2026-03-23T19:58:51.892 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81638.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.81638.log.gz 2026-03-23T19:58:51.892 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81709.log 2026-03-23T19:58:51.893 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81664.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81664.log.gz 2026-03-23T19:58:51.894 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81736.log 2026-03-23T19:58:51.894 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81709.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.81709.log.gz 2026-03-23T19:58:51.894 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81763.log 2026-03-23T19:58:51.895 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81736.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.81736.log.gz 2026-03-23T19:58:51.895 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81790.log 2026-03-23T19:58:51.895 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81763.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.81763.log.gz 2026-03-23T19:58:51.896 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81816.log 2026-03-23T19:58:51.896 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81848.log 2026-03-23T19:58:51.896 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.81790.log.gz 2026-03-23T19:58:51.897 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81816.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81816.log.gz 2026-03-23T19:58:51.897 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81874.log 2026-03-23T19:58:51.898 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81848.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81898.log 2026-03-23T19:58:51.899 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81848.log.gz 2026-03-23T19:58:51.899 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.81874.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81874.log.gz 2026-03-23T19:58:51.900 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81922.log 2026-03-23T19:58:51.900 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.81898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.81949.log 2026-03-23T19:58:51.901 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81898.log.gz 2026-03-23T19:58:51.901 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81922.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.81922.log.gz 2026-03-23T19:58:51.902 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81976.log 2026-03-23T19:58:51.902 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.81949.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.81949.log.gz 2026-03-23T19:58:51.902 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82003.log 2026-03-23T19:58:51.903 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.81976.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82029.log 2026-03-23T19:58:51.903 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.81976.log.gz 2026-03-23T19:58:51.904 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82074.log 2026-03-23T19:58:51.904 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82003.log.gz 2026-03-23T19:58:51.904 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82029.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82101.log 2026-03-23T19:58:51.905 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82029.log.gz 2026-03-23T19:58:51.905 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82074.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82128.log 2026-03-23T19:58:51.906 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82074.log.gz 2026-03-23T19:58:51.906 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82101.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82101.log.gz 2026-03-23T19:58:51.906 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82155.log 2026-03-23T19:58:51.907 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82128.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.82128.log.gz 2026-03-23T19:58:51.907 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82181.log 2026-03-23T19:58:51.908 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82155.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.82155.log.gz 2026-03-23T19:58:51.908 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82237.log 2026-03-23T19:58:51.908 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82263.log 2026-03-23T19:58:51.909 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82181.log.gz 2026-03-23T19:58:51.910 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82237.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82237.log.gz 2026-03-23T19:58:51.911 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82287.log 2026-03-23T19:58:51.911 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82263.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82263.log.gz 2026-03-23T19:58:51.912 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82311.log 2026-03-23T19:58:51.912 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82287.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82287.log.gz 2026-03-23T19:58:51.912 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82338.log 2026-03-23T19:58:51.913 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82311.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.82311.log.gz 2026-03-23T19:58:51.913 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82365.log 2026-03-23T19:58:51.914 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82338.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.82338.log.gz 2026-03-23T19:58:51.914 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82392.log 2026-03-23T19:58:51.914 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82365.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.82365.log.gz 2026-03-23T19:58:51.915 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82418.log 2026-03-23T19:58:51.915 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82392.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.82392.log.gz 2026-03-23T19:58:51.915 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82464.log 2026-03-23T19:58:51.916 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82418.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.82418.log.gz 2026-03-23T19:58:51.916 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82491.log 2026-03-23T19:58:51.917 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82518.log 2026-03-23T19:58:51.917 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82464.log.gz 2026-03-23T19:58:51.918 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82491.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82491.log.gz 2026-03-23T19:58:51.918 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82545.log 2026-03-23T19:58:51.918 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82518.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.82518.log.gz 2026-03-23T19:58:51.919 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82571.log 2026-03-23T19:58:51.919 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82545.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.82545.log.gz 2026-03-23T19:58:51.919 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82603.log 2026-03-23T19:58:51.920 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82571.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82571.log.gz 2026-03-23T19:58:51.920 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82629.log 2026-03-23T19:58:51.921 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82603.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82603.log.gz 2026-03-23T19:58:51.921 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82653.log 2026-03-23T19:58:51.922 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82677.log 2026-03-23T19:58:51.922 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82629.log.gz 2026-03-23T19:58:51.923 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82653.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82653.log.gz 2026-03-23T19:58:51.923 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82704.log 2026-03-23T19:58:51.923 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82677.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82731.log 2026-03-23T19:58:51.924 INFO:teuthology.orchestra.run.vm08.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.82677.log.gz 2026-03-23T19:58:51.924 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82704.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.82704.log.gz 2026-03-23T19:58:51.924 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82758.log 2026-03-23T19:58:51.925 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82784.log 2026-03-23T19:58:51.925 INFO:teuthology.orchestra.run.vm08.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.82731.log.gz 2026-03-23T19:58:51.926 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82758.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82758.log.gz 2026-03-23T19:58:51.926 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82829.log 2026-03-23T19:58:51.927 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.82784.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82784.log.gz 2026-03-23T19:58:51.927 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82856.log 2026-03-23T19:58:51.927 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82829.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82829.log.gz 2026-03-23T19:58:51.928 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82883.log 2026-03-23T19:58:51.928 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.82856.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82856.log.gz 2026-03-23T19:58:51.928 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82910.log 2026-03-23T19:58:51.929 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82883.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.82883.log.gz 2026-03-23T19:58:51.929 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82936.log 2026-03-23T19:58:51.930 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.82910.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.82910.log.gz 2026-03-23T19:58:51.930 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82968.log 2026-03-23T19:58:51.931 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82936.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82936.log.gz 2026-03-23T19:58:51.931 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82994.log 2026-03-23T19:58:51.932 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82968.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82968.log.gz 2026-03-23T19:58:51.932 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83018.log 2026-03-23T19:58:51.933 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.82994.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82994.log.gz 2026-03-23T19:58:51.933 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83042.log 2026-03-23T19:58:51.934 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83018.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83018.log.gz 2026-03-23T19:58:51.934 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83069.log 2026-03-23T19:58:51.935 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83042.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.83042.log.gz 2026-03-23T19:58:51.935 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83096.log 2026-03-23T19:58:51.935 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83069.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.83069.log.gz 2026-03-23T19:58:51.936 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83123.log 2026-03-23T19:58:51.936 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83096.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.83096.log.gz 2026-03-23T19:58:51.937 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83149.log 2026-03-23T19:58:51.938 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83123.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.83123.log.gz 2026-03-23T19:58:51.938 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83194.log 2026-03-23T19:58:51.939 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83149.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.83149.log.gz 2026-03-23T19:58:51.939 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83221.log 2026-03-23T19:58:51.939 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83194.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83194.log.gz 2026-03-23T19:58:51.940 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83248.log 2026-03-23T19:58:51.940 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83221.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83221.log.gz 2026-03-23T19:58:51.940 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83275.log 2026-03-23T19:58:51.941 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83301.log 2026-03-23T19:58:51.941 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.83248.log.gz 2026-03-23T19:58:51.942 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83275.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.83275.log.gz 2026-03-23T19:58:51.942 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83333.log 2026-03-23T19:58:51.943 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83301.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83301.log.gz 2026-03-23T19:58:51.943 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83359.log 2026-03-23T19:58:51.944 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83333.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83333.log.gz 2026-03-23T19:58:51.944 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83383.log 2026-03-23T19:58:51.944 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83407.log 2026-03-23T19:58:51.945 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83434.log 2026-03-23T19:58:51.945 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83407.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83383.log.gz 2026-03-23T19:58:51.946 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83359.log.gz 2026-03-23T19:58:51.946 INFO:teuthology.orchestra.run.vm08.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.83407.log.gz 2026-03-23T19:58:51.946 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83461.log 2026-03-23T19:58:51.948 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83434.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.83434.log.gz 2026-03-23T19:58:51.948 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83488.log 2026-03-23T19:58:51.948 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83514.log 2026-03-23T19:58:51.948 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83461.log.gz 2026-03-23T19:58:51.948 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83488.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83488.log.gz 2026-03-23T19:58:51.949 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83559.log 2026-03-23T19:58:51.950 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83514.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83514.log.gz 2026-03-23T19:58:51.950 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83586.log 2026-03-23T19:58:51.951 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83559.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83559.log.gz 2026-03-23T19:58:51.951 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83613.log 2026-03-23T19:58:51.951 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83586.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83586.log.gz 2026-03-23T19:58:51.952 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83640.log 2026-03-23T19:58:51.952 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83613.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.83613.log.gz 2026-03-23T19:58:51.953 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83666.log 2026-03-23T19:58:51.953 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83640.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.83640.log.gz 2026-03-23T19:58:51.953 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83698.log 2026-03-23T19:58:51.954 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83666.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83666.log.gz 2026-03-23T19:58:51.954 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83724.log 2026-03-23T19:58:51.955 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83698.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83698.log.gz 2026-03-23T19:58:51.955 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83748.log 2026-03-23T19:58:51.956 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83724.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83724.log.gz 2026-03-23T19:58:51.956 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83772.log 2026-03-23T19:58:51.957 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.83748.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83748.log.gz 2026-03-23T19:58:51.957 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83799.log 2026-03-23T19:58:51.958 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83772.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.83772.log.gz 2026-03-23T19:58:51.958 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83826.log 2026-03-23T19:58:51.958 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83799.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83799.log.gz 2026-03-23T19:58:51.959 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83853.log 2026-03-23T19:58:51.959 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83826.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.83826.log.gz 2026-03-23T19:58:51.959 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83879.log 2026-03-23T19:58:51.960 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83853.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.83853.log.gz 2026-03-23T19:58:51.960 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83924.log 2026-03-23T19:58:51.961 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.83879.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.83879.log.gz 2026-03-23T19:58:51.961 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83951.log 2026-03-23T19:58:51.962 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83924.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83924.log.gz 2026-03-23T19:58:51.962 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83978.log 2026-03-23T19:58:51.962 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.83951.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83951.log.gz 2026-03-23T19:58:51.963 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84005.log 2026-03-23T19:58:51.963 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.83978.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.83978.log.gz 2026-03-23T19:58:51.963 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84031.log 2026-03-23T19:58:51.964 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84005.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.84005.log.gz 2026-03-23T19:58:51.964 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84063.log 2026-03-23T19:58:51.965 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84031.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84031.log.gz 2026-03-23T19:58:51.965 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84089.log 2026-03-23T19:58:51.965 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84113.log 2026-03-23T19:58:51.966 INFO:teuthology.orchestra.run.vm08.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84063.log.gz 2026-03-23T19:58:51.967 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84089.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84089.log.gz 2026-03-23T19:58:51.967 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84137.log 2026-03-23T19:58:51.967 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84113.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84113.log.gz 2026-03-23T19:58:51.968 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84164.log 2026-03-23T19:58:51.968 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84137.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.84137.log.gz 2026-03-23T19:58:51.968 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84191.log 2026-03-23T19:58:51.969 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84164.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.84164.log.gz 2026-03-23T19:58:51.969 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84218.log 2026-03-23T19:58:51.970 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84191.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.84191.log.gz 2026-03-23T19:58:51.970 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84244.log 2026-03-23T19:58:51.970 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84289.log 2026-03-23T19:58:51.970 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.84218.log.gz 2026-03-23T19:58:51.971 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84316.log 2026-03-23T19:58:51.971 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84244.log.gz 2026-03-23T19:58:51.972 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84343.log 2026-03-23T19:58:51.972 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.84289.log.gz 2026-03-23T19:58:51.973 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84316.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.84316.log.gz 2026-03-23T19:58:51.973 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84370.log 2026-03-23T19:58:51.973 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84396.log 2026-03-23T19:58:51.974 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.84343.log.gz 2026-03-23T19:58:51.974 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84370.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.84370.log.gz 2026-03-23T19:58:51.974 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84428.log 2026-03-23T19:58:51.975 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84396.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84396.log.gz 2026-03-23T19:58:51.976 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84455.log 2026-03-23T19:58:51.976 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84428.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84428.log.gz 2026-03-23T19:58:51.977 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84479.log 2026-03-23T19:58:51.978 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84455.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84455.log.gz 2026-03-23T19:58:51.978 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84503.log 2026-03-23T19:58:51.978 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84479.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84479.log.gz 2026-03-23T19:58:51.979 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84530.log 2026-03-23T19:58:51.979 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84503.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.84503.log.gz 2026-03-23T19:58:51.979 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84557.log 2026-03-23T19:58:51.980 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84530.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.84530.log.gz 2026-03-23T19:58:51.980 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84584.log 2026-03-23T19:58:51.981 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84557.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.84557.log.gz 2026-03-23T19:58:51.981 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84610.log 2026-03-23T19:58:51.981 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84584.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.84584.log.gz 2026-03-23T19:58:51.982 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84655.log 2026-03-23T19:58:51.983 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84610.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84610.log.gz 2026-03-23T19:58:51.983 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84682.log 2026-03-23T19:58:51.983 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84655.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.84655.log.gz 2026-03-23T19:58:51.983 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84709.log 2026-03-23T19:58:51.984 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84682.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.84682.log.gz 2026-03-23T19:58:51.984 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84736.log 2026-03-23T19:58:51.985 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84762.log 2026-03-23T19:58:51.985 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.84709.log.gz 2026-03-23T19:58:51.986 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84794.log 2026-03-23T19:58:51.986 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.84736.log.gz 2026-03-23T19:58:51.987 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84820.log 2026-03-23T19:58:51.987 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84794.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84762.log.gz 2026-03-23T19:58:51.988 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84844.log 2026-03-23T19:58:51.989 INFO:teuthology.orchestra.run.vm08.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84794.log.gz 2026-03-23T19:58:51.989 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84820.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84820.log.gz 2026-03-23T19:58:51.990 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84868.log 2026-03-23T19:58:51.990 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.84844.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84844.log.gz 2026-03-23T19:58:51.990 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84895.log 2026-03-23T19:58:51.991 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84868.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.84868.log.gz 2026-03-23T19:58:51.991 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84922.log 2026-03-23T19:58:51.992 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.84895.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.84895.log.gz 2026-03-23T19:58:51.992 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84949.log 2026-03-23T19:58:51.993 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84922.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.84922.log.gz 2026-03-23T19:58:51.993 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84975.log 2026-03-23T19:58:51.994 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.84949.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.84949.log.gz 2026-03-23T19:58:51.994 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85020.log 2026-03-23T19:58:51.995 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.84975.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84975.log.gz 2026-03-23T19:58:51.995 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85047.log 2026-03-23T19:58:51.995 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85020.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85020.log.gz 2026-03-23T19:58:51.995 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85074.log 2026-03-23T19:58:51.996 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85047.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.85047.log.gz 2026-03-23T19:58:51.996 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85101.log 2026-03-23T19:58:51.996 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85127.log 2026-03-23T19:58:51.997 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.85074.log.gz 2026-03-23T19:58:51.997 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85101.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.85101.log.gz 2026-03-23T19:58:51.997 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85171.log 2026-03-23T19:58:51.998 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85127.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.85127.log.gz 2026-03-23T19:58:51.999 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85197.log 2026-03-23T19:58:51.999 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85171.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85221.log 2026-03-23T19:58:52.000 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85197.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85171.log.gz 2026-03-23T19:58:52.001 INFO:teuthology.orchestra.run.vm08.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85197.log.gz 2026-03-23T19:58:52.001 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85245.log 2026-03-23T19:58:52.001 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85221.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85221.log.gz 2026-03-23T19:58:52.002 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85272.log 2026-03-23T19:58:52.002 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85245.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.85245.log.gz 2026-03-23T19:58:52.003 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85299.log 2026-03-23T19:58:52.003 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85272.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85272.log.gz 2026-03-23T19:58:52.003 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85326.log 2026-03-23T19:58:52.004 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85299.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.85299.log.gz 2026-03-23T19:58:52.004 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85352.log 2026-03-23T19:58:52.004 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85326.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.85326.log.gz 2026-03-23T19:58:52.005 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85397.log 2026-03-23T19:58:52.006 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85352.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.85352.log.gz 2026-03-23T19:58:52.006 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85424.log 2026-03-23T19:58:52.006 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85397.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85397.log.gz 2026-03-23T19:58:52.006 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85451.log 2026-03-23T19:58:52.007 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85478.log 2026-03-23T19:58:52.007 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.85424.log.gz 2026-03-23T19:58:52.008 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85504.log 2026-03-23T19:58:52.008 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.85451.log.gz 2026-03-23T19:58:52.008 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85536.log 2026-03-23T19:58:52.009 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.85478.log.gz 2026-03-23T19:58:52.009 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85562.log 2026-03-23T19:58:52.010 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85504.log.gz 2026-03-23T19:58:52.011 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85536.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85536.log.gz 2026-03-23T19:58:52.011 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85586.log 2026-03-23T19:58:52.011 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85562.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85562.log.gz 2026-03-23T19:58:52.011 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85610.log 2026-03-23T19:58:52.012 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85586.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85586.log.gz 2026-03-23T19:58:52.012 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85634.log 2026-03-23T19:58:52.013 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85610.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85610.log.gz 2026-03-23T19:58:52.013 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85658.log 2026-03-23T19:58:52.013 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85634.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85634.log.gz 2026-03-23T19:58:52.014 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85685.log 2026-03-23T19:58:52.014 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85658.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.85658.log.gz 2026-03-23T19:58:52.014 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85712.log 2026-03-23T19:58:52.015 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85685.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.85685.log.gz 2026-03-23T19:58:52.015 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85739.log 2026-03-23T19:58:52.016 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85712.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.85712.log.gz 2026-03-23T19:58:52.016 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85765.log 2026-03-23T19:58:52.016 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85739.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85739.log.gz 2026-03-23T19:58:52.016 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85798.log 2026-03-23T19:58:52.018 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85765.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85765.log.gz 2026-03-23T19:58:52.018 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85825.log 2026-03-23T19:58:52.018 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85798.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85798.log.gz 2026-03-23T19:58:52.019 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85852.log 2026-03-23T19:58:52.019 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.85825.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85825.log.gz 2026-03-23T19:58:52.019 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85879.log 2026-03-23T19:58:52.020 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85852.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85905.log 2026-03-23T19:58:52.020 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.85879.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.85852.log.gz 2026-03-23T19:58:52.021 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.85879.log.gz 2026-03-23T19:58:52.021 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85937.log 2026-03-23T19:58:52.021 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85905.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85905.log.gz 2026-03-23T19:58:52.022 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85963.log 2026-03-23T19:58:52.023 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85937.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85937.log.gz 2026-03-23T19:58:52.023 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85987.log 2026-03-23T19:58:52.023 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.85963.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.85963.log.gz 2026-03-23T19:58:52.024 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86011.log 2026-03-23T19:58:52.024 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.85987.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85987.log.gz 2026-03-23T19:58:52.024 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86038.log 2026-03-23T19:58:52.025 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86011.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.86011.log.gz 2026-03-23T19:58:52.025 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86065.log 2026-03-23T19:58:52.026 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86038.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.86038.log.gz 2026-03-23T19:58:52.026 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86092.log 2026-03-23T19:58:52.027 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86065.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.86065.log.gz 2026-03-23T19:58:52.027 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86118.log 2026-03-23T19:58:52.027 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86163.log 2026-03-23T19:58:52.028 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86118.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86092.log.gz 2026-03-23T19:58:52.028 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.86118.log.gz 2026-03-23T19:58:52.029 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86190.log 2026-03-23T19:58:52.029 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86163.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.86163.log.gz 2026-03-23T19:58:52.029 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86217.log 2026-03-23T19:58:52.030 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86190.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.86190.log.gz 2026-03-23T19:58:52.030 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86244.log 2026-03-23T19:58:52.031 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86217.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.86217.log.gz 2026-03-23T19:58:52.031 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86270.log 2026-03-23T19:58:52.031 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86244.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86244.log.gz 2026-03-23T19:58:52.032 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86314.log 2026-03-23T19:58:52.033 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.86270.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86270.log.gz 2026-03-23T19:58:52.033 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86340.log 2026-03-23T19:58:52.034 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.86314.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86314.log.gz 2026-03-23T19:58:52.034 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86364.log 2026-03-23T19:58:52.035 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.86340.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86340.log.gz 2026-03-23T19:58:52.035 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86388.log 2026-03-23T19:58:52.036 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86364.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86364.log.gz 2026-03-23T19:58:52.036 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86415.log 2026-03-23T19:58:52.036 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86388.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.86388.log.gz 2026-03-23T19:58:52.037 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86442.log 2026-03-23T19:58:52.037 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86415.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.86415.log.gz 2026-03-23T19:58:52.037 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86469.log 2026-03-23T19:58:52.038 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86442.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86442.log.gz 2026-03-23T19:58:52.038 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86495.log 2026-03-23T19:58:52.038 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86540.log 2026-03-23T19:58:52.039 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.86469.log.gz 2026-03-23T19:58:52.040 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.86495.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86495.log.gz 2026-03-23T19:58:52.040 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86567.log 2026-03-23T19:58:52.040 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86540.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86540.log.gz 2026-03-23T19:58:52.041 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86594.log 2026-03-23T19:58:52.041 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86567.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.86567.log.gz 2026-03-23T19:58:52.041 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86621.log 2026-03-23T19:58:52.042 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86594.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.86594.log.gz 2026-03-23T19:58:52.042 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86647.log 2026-03-23T19:58:52.043 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86621.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.86621.log.gz 2026-03-23T19:58:52.043 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86679.log 2026-03-23T19:58:52.043 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86647.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.86647.log.gz 2026-03-23T19:58:52.044 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86705.log 2026-03-23T19:58:52.045 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86679.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.86679.log.gz 2026-03-23T19:58:52.045 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86729.log 2026-03-23T19:58:52.046 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86705.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86705.log.gz 2026-03-23T19:58:52.046 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86753.log 2026-03-23T19:58:52.046 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.86729.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86729.log.gz 2026-03-23T19:58:52.047 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86780.log 2026-03-23T19:58:52.047 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86753.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.86753.log.gz 2026-03-23T19:58:52.047 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86807.log 2026-03-23T19:58:52.048 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86834.log 2026-03-23T19:58:52.048 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.86780.log.gz 2026-03-23T19:58:52.049 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86807.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.86807.log.gz 2026-03-23T19:58:52.049 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86860.log 2026-03-23T19:58:52.049 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86834.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86834.log.gz 2026-03-23T19:58:52.050 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86905.log 2026-03-23T19:58:52.051 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.86860.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.86860.log.gz 2026-03-23T19:58:52.051 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86932.log 2026-03-23T19:58:52.051 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86905.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.86905.log.gz 2026-03-23T19:58:52.051 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86959.log 2026-03-23T19:58:52.052 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.86932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86986.log 2026-03-23T19:58:52.052 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86959.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86932.log.gz 2026-03-23T19:58:52.053 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86959.log.gz 2026-03-23T19:58:52.053 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87012.log 2026-03-23T19:58:52.053 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.86986.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.86986.log.gz 2026-03-23T19:58:52.054 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87044.log 2026-03-23T19:58:52.054 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87070.log 2026-03-23T19:58:52.055 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87012.log.gz 2026-03-23T19:58:52.056 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87044.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87044.log.gz 2026-03-23T19:58:52.056 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87094.log 2026-03-23T19:58:52.056 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87070.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87070.log.gz 2026-03-23T19:58:52.057 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87118.log 2026-03-23T19:58:52.057 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87094.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87094.log.gz 2026-03-23T19:58:52.057 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87145.log 2026-03-23T19:58:52.058 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87118.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.87118.log.gz 2026-03-23T19:58:52.058 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87172.log 2026-03-23T19:58:52.059 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87145.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.87145.log.gz 2026-03-23T19:58:52.059 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87199.log 2026-03-23T19:58:52.059 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87172.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.87172.log.gz 2026-03-23T19:58:52.060 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87225.log 2026-03-23T19:58:52.060 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87199.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.87199.log.gz 2026-03-23T19:58:52.060 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87270.log 2026-03-23T19:58:52.061 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87225.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87225.log.gz 2026-03-23T19:58:52.061 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87297.log 2026-03-23T19:58:52.062 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87270.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.87270.log.gz 2026-03-23T19:58:52.062 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87324.log 2026-03-23T19:58:52.063 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87297.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.87297.log.gz 2026-03-23T19:58:52.063 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87351.log 2026-03-23T19:58:52.063 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87324.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.87324.log.gz 2026-03-23T19:58:52.064 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87377.log 2026-03-23T19:58:52.064 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87351.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.87351.log.gz 2026-03-23T19:58:52.064 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87433.log 2026-03-23T19:58:52.065 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87377.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87377.log.gz 2026-03-23T19:58:52.065 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87459.log 2026-03-23T19:58:52.066 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87433.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87433.log.gz 2026-03-23T19:58:52.067 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87483.log 2026-03-23T19:58:52.067 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87459.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87459.log.gz 2026-03-23T19:58:52.067 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87507.log 2026-03-23T19:58:52.068 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87483.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87483.log.gz 2026-03-23T19:58:52.068 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87534.log 2026-03-23T19:58:52.069 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87507.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.87507.log.gz 2026-03-23T19:58:52.069 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87561.log 2026-03-23T19:58:52.069 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87534.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.87534.log.gz 2026-03-23T19:58:52.070 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87588.log 2026-03-23T19:58:52.070 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87561.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.87561.log.gz 2026-03-23T19:58:52.070 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87614.log 2026-03-23T19:58:52.071 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87588.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.87588.log.gz 2026-03-23T19:58:52.071 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87659.log 2026-03-23T19:58:52.072 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87614.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.87614.log.gz 2026-03-23T19:58:52.072 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87686.log 2026-03-23T19:58:52.073 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87659.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.87659.log.gz 2026-03-23T19:58:52.073 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87713.log 2026-03-23T19:58:52.073 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87686.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.87686.log.gz 2026-03-23T19:58:52.074 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87740.log 2026-03-23T19:58:52.074 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87713.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.87713.log.gz 2026-03-23T19:58:52.074 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87766.log 2026-03-23T19:58:52.075 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87740.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.87740.log.gz 2026-03-23T19:58:52.075 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87798.log 2026-03-23T19:58:52.076 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87766.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87766.log.gz 2026-03-23T19:58:52.076 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87824.log 2026-03-23T19:58:52.077 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87798.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87848.log 2026-03-23T19:58:52.078 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87798.log.gz 2026-03-23T19:58:52.079 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87824.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87824.log.gz 2026-03-23T19:58:52.079 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87872.log 2026-03-23T19:58:52.079 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.87848.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87848.log.gz 2026-03-23T19:58:52.080 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87899.log 2026-03-23T19:58:52.080 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87872.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.87872.log.gz 2026-03-23T19:58:52.080 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87926.log 2026-03-23T19:58:52.081 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.87899.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.87899.log.gz 2026-03-23T19:58:52.081 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87953.log 2026-03-23T19:58:52.082 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87926.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.87926.log.gz 2026-03-23T19:58:52.082 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87979.log 2026-03-23T19:58:52.082 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.87953.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.87953.log.gz 2026-03-23T19:58:52.083 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88024.log 2026-03-23T19:58:52.083 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.87979.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87979.log.gz 2026-03-23T19:58:52.084 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88051.log 2026-03-23T19:58:52.084 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88024.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.88024.log.gz 2026-03-23T19:58:52.084 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88078.log 2026-03-23T19:58:52.085 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88051.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.88051.log.gz 2026-03-23T19:58:52.085 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88105.log 2026-03-23T19:58:52.085 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88078.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.88078.log.gz 2026-03-23T19:58:52.086 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88131.log 2026-03-23T19:58:52.086 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88105.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88163.log 2026-03-23T19:58:52.086 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.88105.log.gz 2026-03-23T19:58:52.087 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88131.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88131.log.gz 2026-03-23T19:58:52.087 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88189.log 2026-03-23T19:58:52.088 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88163.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.88163.log.gz 2026-03-23T19:58:52.088 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88213.log 2026-03-23T19:58:52.089 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88189.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88189.log.gz 2026-03-23T19:58:52.089 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88237.log 2026-03-23T19:58:52.089 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88213.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88213.log.gz 2026-03-23T19:58:52.090 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88264.log 2026-03-23T19:58:52.090 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88237.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.88237.log.gz 2026-03-23T19:58:52.090 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88291.log 2026-03-23T19:58:52.091 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88264.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.88264.log.gz 2026-03-23T19:58:52.091 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88318.log 2026-03-23T19:58:52.092 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88291.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.88291.log.gz 2026-03-23T19:58:52.092 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88344.log 2026-03-23T19:58:52.092 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88318.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.88318.log.gz 2026-03-23T19:58:52.093 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88377.log 2026-03-23T19:58:52.093 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88399.log 2026-03-23T19:58:52.093 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88344.log.gz 2026-03-23T19:58:52.094 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88426.log 2026-03-23T19:58:52.094 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.88377.log.gz 2026-03-23T19:58:52.095 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88399.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88453.log 2026-03-23T19:58:52.095 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.88399.log.gz 2026-03-23T19:58:52.096 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88480.log 2026-03-23T19:58:52.096 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.88426.log.gz 2026-03-23T19:58:52.096 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88453.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.88453.log.gz 2026-03-23T19:58:52.097 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88506.log 2026-03-23T19:58:52.097 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88480.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.88480.log.gz 2026-03-23T19:58:52.097 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88538.log 2026-03-23T19:58:52.098 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88564.log 2026-03-23T19:58:52.098 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88506.log.gz 2026-03-23T19:58:52.099 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88588.log 2026-03-23T19:58:52.099 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88538.log.gz 2026-03-23T19:58:52.100 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88564.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88564.log.gz 2026-03-23T19:58:52.101 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88612.log 2026-03-23T19:58:52.101 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88588.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.88588.log.gz 2026-03-23T19:58:52.101 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88639.log 2026-03-23T19:58:52.102 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88612.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.88612.log.gz 2026-03-23T19:58:52.103 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88666.log 2026-03-23T19:58:52.104 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.88639.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.88639.log.gz 2026-03-23T19:58:52.104 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88693.log 2026-03-23T19:58:52.105 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88666.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.88666.log.gz 2026-03-23T19:58:52.105 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88720.log 2026-03-23T19:58:52.106 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.88693.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.88693.log.gz 2026-03-23T19:58:52.106 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88742.log 2026-03-23T19:58:52.106 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88720.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88720.log.gz 2026-03-23T19:58:52.107 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88766.log 2026-03-23T19:58:52.107 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88742.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88742.log.gz 2026-03-23T19:58:52.108 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88783.log 2026-03-23T19:58:52.108 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88766.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88766.log.gz 2026-03-23T19:58:52.108 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88803.log 2026-03-23T19:58:52.109 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88783.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88783.log.gz 2026-03-23T19:58:52.109 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88823.log 2026-03-23T19:58:52.110 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88803.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88803.log.gz 2026-03-23T19:58:52.112 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88896.log 2026-03-23T19:58:52.112 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88922.log 2026-03-23T19:58:52.116 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88823.log.gz 2026-03-23T19:58:52.117 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88896.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88896.log.gz 2026-03-23T19:58:52.117 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88946.log 2026-03-23T19:58:52.118 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.88922.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88922.log.gz 2026-03-23T19:58:52.119 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88970.log 2026-03-23T19:58:52.119 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88946.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88946.log.gz 2026-03-23T19:58:52.120 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88994.log 2026-03-23T19:58:52.120 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88970.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88970.log.gz 2026-03-23T19:58:52.120 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89018.log 2026-03-23T19:58:52.121 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.88994.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.88994.log.gz 2026-03-23T19:58:52.121 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89045.log 2026-03-23T19:58:52.122 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89018.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.89018.log.gz 2026-03-23T19:58:52.122 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89072.log 2026-03-23T19:58:52.122 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89099.log 2026-03-23T19:58:52.122 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.89045.log.gz 2026-03-23T19:58:52.124 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.89072.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89125.log 2026-03-23T19:58:52.125 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.89099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89179.log 2026-03-23T19:58:52.125 INFO:teuthology.orchestra.run.vm08.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.89072.log.gz 2026-03-23T19:58:52.125 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.89099.log.gz 2026-03-23T19:58:52.125 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89206.log 2026-03-23T19:58:52.126 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89179.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.89179.log.gz 2026-03-23T19:58:52.126 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89233.log 2026-03-23T19:58:52.127 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89206.log: /var/log/ceph/cluster1-client.mirror.89125.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.89206.log.gz 2026-03-23T19:58:52.127 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89260.log 2026-03-23T19:58:52.127 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.89233.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.89233.log.gz 2026-03-23T19:58:52.128 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89286.log 2026-03-23T19:58:52.128 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.89260.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.89260.log.gz 2026-03-23T19:58:52.129 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89307.log 2026-03-23T19:58:52.133 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89286.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.89286.log.gz 2026-03-23T19:58:52.133 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89332.log 2026-03-23T19:58:52.137 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89307.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89307.log.gz 2026-03-23T19:58:52.137 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89352.log 2026-03-23T19:58:52.137 INFO:teuthology.orchestra.run.vm08.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89125.log.gz 2026-03-23T19:58:52.138 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89332.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89332.log.gz 2026-03-23T19:58:52.138 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89374.log 2026-03-23T19:58:52.139 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89352.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.89352.log.gz 2026-03-23T19:58:52.139 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89394.log 2026-03-23T19:58:52.140 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89374.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89374.log.gz 2026-03-23T19:58:52.140 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89421.log 2026-03-23T19:58:52.142 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89394.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89394.log.gz 2026-03-23T19:58:52.142 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89438.log 2026-03-23T19:58:52.142 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89421.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89421.log.gz 2026-03-23T19:58:52.143 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89458.log 2026-03-23T19:58:52.143 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89478.log 2026-03-23T19:58:52.144 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89438.log.gz 2026-03-23T19:58:52.144 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89458.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89458.log.gz 2026-03-23T19:58:52.144 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89547.log 2026-03-23T19:58:52.145 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89567.log 2026-03-23T19:58:52.148 INFO:teuthology.orchestra.run.vm08.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89478.log.gz 2026-03-23T19:58:52.149 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89547.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89547.log.gz 2026-03-23T19:58:52.149 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89587.log 2026-03-23T19:58:52.150 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89567.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89567.log.gz 2026-03-23T19:58:52.150 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89607.log 2026-03-23T19:58:52.150 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89587.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89587.log.gz 2026-03-23T19:58:52.151 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89628.log 2026-03-23T19:58:52.157 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89607.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89607.log.gz 2026-03-23T19:58:52.157 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89651.log 2026-03-23T19:58:52.158 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89628.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89628.log.gz 2026-03-23T19:58:52.158 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89688.log 2026-03-23T19:58:52.158 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89651.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89651.log.gz 2026-03-23T19:58:52.159 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89714.log 2026-03-23T19:58:52.159 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89738.log 2026-03-23T19:58:52.160 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89688.log.gz 2026-03-23T19:58:52.160 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89762.log 2026-03-23T19:58:52.160 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89714.log.gz 2026-03-23T19:58:52.161 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89738.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89738.log.gz 2026-03-23T19:58:52.161 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89786.log 2026-03-23T19:58:52.161 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89762.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89762.log.gz 2026-03-23T19:58:52.162 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89810.log 2026-03-23T19:58:52.162 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89786.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89786.log.gz 2026-03-23T19:58:52.162 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89834.log 2026-03-23T19:58:52.163 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89810.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89810.log.gz 2026-03-23T19:58:52.163 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89861.log 2026-03-23T19:58:52.164 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89834.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.admin.89834.log.gz 2026-03-23T19:58:52.164 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89894.log 2026-03-23T19:58:52.164 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.89861.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.89861.log.gz 2026-03-23T19:58:52.165 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89914.log 2026-03-23T19:58:52.165 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89952.log 2026-03-23T19:58:52.166 INFO:teuthology.orchestra.run.vm08.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89894.log.gz 2026-03-23T19:58:52.167 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.89914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89978.log 2026-03-23T19:58:52.168 INFO:teuthology.orchestra.run.vm08.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.89914.log.gz 2026-03-23T19:58:52.169 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89952.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89952.log.gz 2026-03-23T19:58:52.169 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90002.log 2026-03-23T19:58:52.169 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.89978.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90026.log 2026-03-23T19:58:52.170 INFO:teuthology.orchestra.run.vm08.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89978.log.gz 2026-03-23T19:58:52.170 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90002.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.90002.log.gz 2026-03-23T19:58:52.171 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90050.log 2026-03-23T19:58:52.171 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90026.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.90026.log.gz 2026-03-23T19:58:52.171 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90077.log 2026-03-23T19:58:52.172 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.90050.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.90050.log.gz 2026-03-23T19:58:52.172 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90110.log 2026-03-23T19:58:52.173 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.90077.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.90077.log.gz 2026-03-23T19:58:52.173 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90130.log 2026-03-23T19:58:52.174 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90153.log 2026-03-23T19:58:52.175 INFO:teuthology.orchestra.run.vm08.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90110.log.gz 2026-03-23T19:58:52.176 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90173.log 2026-03-23T19:58:52.177 INFO:teuthology.orchestra.run.vm08.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90130.log.gz 2026-03-23T19:58:52.177 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90153.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90153.log.gz 2026-03-23T19:58:52.177 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90197.log 2026-03-23T19:58:52.179 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90173.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90173.log.gz 2026-03-23T19:58:52.179 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90217.log 2026-03-23T19:58:52.180 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90197.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90197.log.gz 2026-03-23T19:58:52.180 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90237.log 2026-03-23T19:58:52.181 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90217.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90217.log.gz 2026-03-23T19:58:52.181 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90257.log 2026-03-23T19:58:52.182 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90237.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.90237.log.gz 2026-03-23T19:58:52.182 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90277.log 2026-03-23T19:58:52.183 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90257.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90257.log.gz 2026-03-23T19:58:52.183 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90297.log 2026-03-23T19:58:52.184 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90277.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90277.log.gz 2026-03-23T19:58:52.184 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90318.log 2026-03-23T19:58:52.185 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90297.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.90297.log.gz 2026-03-23T19:58:52.185 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90382.log 2026-03-23T19:58:52.186 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90318.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90318.log.gz 2026-03-23T19:58:52.186 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90404.log 2026-03-23T19:58:52.187 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90382.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.90382.log.gz 2026-03-23T19:58:52.187 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90424.log 2026-03-23T19:58:52.188 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90404.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90404.log.gz 2026-03-23T19:58:52.188 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90446.log 2026-03-23T19:58:52.190 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90424.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90424.log.gz 2026-03-23T19:58:52.190 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90466.log 2026-03-23T19:58:52.190 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90446.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.90446.log.gz 2026-03-23T19:58:52.191 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90488.log 2026-03-23T19:58:52.192 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90466.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.90466.log.gz 2026-03-23T19:58:52.193 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90510.log 2026-03-23T19:58:52.193 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90488.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90488.log.gz 2026-03-23T19:58:52.193 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90530.log 2026-03-23T19:58:52.194 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90510.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90510.log.gz 2026-03-23T19:58:52.194 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90553.log 2026-03-23T19:58:52.195 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90530.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90530.log.gz 2026-03-23T19:58:52.195 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90573.log 2026-03-23T19:58:52.196 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90553.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90553.log.gz 2026-03-23T19:58:52.196 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90593.log 2026-03-23T19:58:52.197 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90573.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90573.log.gz 2026-03-23T19:58:52.197 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90613.log 2026-03-23T19:58:52.198 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90593.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90593.log.gz 2026-03-23T19:58:52.198 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90633.log 2026-03-23T19:58:52.199 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90613.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90654.log 2026-03-23T19:58:52.199 INFO:teuthology.orchestra.run.vm08.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90613.log.gz 2026-03-23T19:58:52.200 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90633.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90633.log.gz 2026-03-23T19:58:52.200 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90710.log 2026-03-23T19:58:52.201 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90654.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90654.log.gz 2026-03-23T19:58:52.201 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90731.log 2026-03-23T19:58:52.201 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90710.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.90710.log.gz 2026-03-23T19:58:52.202 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90751.log 2026-03-23T19:58:52.202 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90731.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90731.log.gz 2026-03-23T19:58:52.203 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90771.log 2026-03-23T19:58:52.203 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90751.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.90751.log.gz 2026-03-23T19:58:52.204 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90793.log 2026-03-23T19:58:52.204 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90771.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.90771.log.gz 2026-03-23T19:58:52.204 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90815.log 2026-03-23T19:58:52.205 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90793.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90793.log.gz 2026-03-23T19:58:52.205 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90837.log 2026-03-23T19:58:52.206 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90815.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90815.log.gz 2026-03-23T19:58:52.206 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90857.log 2026-03-23T19:58:52.206 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90837.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90837.log.gz 2026-03-23T19:58:52.207 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90879.log 2026-03-23T19:58:52.209 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90857.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90857.log.gz 2026-03-23T19:58:52.209 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90902.log 2026-03-23T19:58:52.209 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90879.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90879.log.gz 2026-03-23T19:58:52.209 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90923.log 2026-03-23T19:58:52.210 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90902.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.90902.log.gz 2026-03-23T19:58:52.210 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90945.log 2026-03-23T19:58:52.211 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90923.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90969.log 2026-03-23T19:58:52.211 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90923.log.gz 2026-03-23T19:58:52.212 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.90945.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.90945.log.gz 2026-03-23T19:58:52.212 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90989.log 2026-03-23T19:58:52.213 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90969.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90969.log.gz 2026-03-23T19:58:52.213 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91010.log 2026-03-23T19:58:52.213 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.90989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91030.log 2026-03-23T19:58:52.214 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91010.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90989.log.gz 2026-03-23T19:58:52.214 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91050.log 2026-03-23T19:58:52.215 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.91010.log.gz 2026-03-23T19:58:52.216 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91030.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91030.log.gz 2026-03-23T19:58:52.216 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91070.log 2026-03-23T19:58:52.216 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91090.log 2026-03-23T19:58:52.217 INFO:teuthology.orchestra.run.vm08.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.91050.log.gz 2026-03-23T19:58:52.218 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91070.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91070.log.gz 2026-03-23T19:58:52.218 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91110.log 2026-03-23T19:58:52.218 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91090.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91090.log.gz 2026-03-23T19:58:52.219 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91131.log 2026-03-23T19:58:52.219 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91110.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91110.log.gz 2026-03-23T19:58:52.219 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91158.log 2026-03-23T19:58:52.220 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91131.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91131.log.gz 2026-03-23T19:58:52.220 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91180.log 2026-03-23T19:58:52.221 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91158.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91158.log.gz 2026-03-23T19:58:52.221 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91200.log 2026-03-23T19:58:52.222 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91180.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.91180.log.gz 2026-03-23T19:58:52.222 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91222.log 2026-03-23T19:58:52.223 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91244.log 2026-03-23T19:58:52.225 INFO:teuthology.orchestra.run.vm08.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.91200.log.gz 2026-03-23T19:58:52.225 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91222.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.91222.log.gz 2026-03-23T19:58:52.225 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91264.log 2026-03-23T19:58:52.226 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91244.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91244.log.gz 2026-03-23T19:58:52.226 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91288.log 2026-03-23T19:58:52.228 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91264.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91264.log.gz 2026-03-23T19:58:52.228 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91305.log 2026-03-23T19:58:52.228 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91288.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91288.log.gz 2026-03-23T19:58:52.228 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91325.log 2026-03-23T19:58:52.229 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91305.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91305.log.gz 2026-03-23T19:58:52.229 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91346.log 2026-03-23T19:58:52.230 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91325.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91325.log.gz 2026-03-23T19:58:52.230 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91403.log 2026-03-23T19:58:52.231 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91346.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91346.log.gz 2026-03-23T19:58:52.231 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91424.log 2026-03-23T19:58:52.232 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91403.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91403.log.gz 2026-03-23T19:58:52.232 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91444.log 2026-03-23T19:58:52.232 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91424.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91424.log.gz 2026-03-23T19:58:52.233 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91466.log 2026-03-23T19:58:52.237 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91444.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91444.log.gz 2026-03-23T19:58:52.237 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91486.log 2026-03-23T19:58:52.238 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91466.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91466.log.gz 2026-03-23T19:58:52.238 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91509.log 2026-03-23T19:58:52.238 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91486.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91535.log 2026-03-23T19:58:52.242 INFO:teuthology.orchestra.run.vm08.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91486.log.gz 2026-03-23T19:58:52.242 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91509.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.91509.log.gz 2026-03-23T19:58:52.243 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91559.log 2026-03-23T19:58:52.243 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91583.log 2026-03-23T19:58:52.243 INFO:teuthology.orchestra.run.vm08.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.91535.log.gz 2026-03-23T19:58:52.244 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91559.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.91559.log.gz 2026-03-23T19:58:52.244 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91607.log 2026-03-23T19:58:52.244 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91583.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91583.log.gz 2026-03-23T19:58:52.245 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91631.log 2026-03-23T19:58:52.245 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91607.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91607.log.gz 2026-03-23T19:58:52.245 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.91655.log 2026-03-23T19:58:52.246 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91631.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.91631.log.gz 2026-03-23T19:58:52.246 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.91682.log 2026-03-23T19:58:52.246 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.91655.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.91655.log.gz 2026-03-23T19:58:52.247 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91715.log 2026-03-23T19:58:52.247 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.91682.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.91682.log.gz 2026-03-23T19:58:52.247 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91735.log 2026-03-23T19:58:52.249 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91715.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91715.log.gz 2026-03-23T19:58:52.249 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91764.log 2026-03-23T19:58:52.249 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91784.log 2026-03-23T19:58:52.250 INFO:teuthology.orchestra.run.vm08.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91735.log.gz 2026-03-23T19:58:52.251 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91764.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91764.log.gz 2026-03-23T19:58:52.252 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91813.log 2026-03-23T19:58:52.253 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91784.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.91784.log.gz 2026-03-23T19:58:52.253 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91833.log 2026-03-23T19:58:52.254 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91813.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91813.log.gz 2026-03-23T19:58:52.254 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91856.log 2026-03-23T19:58:52.255 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.91833.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.91833.log.gz 2026-03-23T19:58:52.256 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91876.log 2026-03-23T19:58:52.256 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91856.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91900.log 2026-03-23T19:58:52.256 INFO:teuthology.orchestra.run.vm08.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91856.log.gz 2026-03-23T19:58:52.258 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91876.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91876.log.gz 2026-03-23T19:58:52.258 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91917.log 2026-03-23T19:58:52.259 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91900.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91900.log.gz 2026-03-23T19:58:52.259 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91937.log 2026-03-23T19:58:52.259 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91957.log 2026-03-23T19:58:52.260 INFO:teuthology.orchestra.run.vm08.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91917.log.gz 2026-03-23T19:58:52.260 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91979.log 2026-03-23T19:58:52.260 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91937.log.gz 2026-03-23T19:58:52.264 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91957.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91957.log.gz 2026-03-23T19:58:52.264 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91999.log 2026-03-23T19:58:52.265 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91979.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91979.log.gz 2026-03-23T19:58:52.265 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92020.log 2026-03-23T19:58:52.265 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.91999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92042.log 2026-03-23T19:58:52.266 INFO:teuthology.orchestra.run.vm08.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91999.log.gz 2026-03-23T19:58:52.266 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92020.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92063.log 2026-03-23T19:58:52.269 INFO:teuthology.orchestra.run.vm08.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92020.log.gz 2026-03-23T19:58:52.270 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92042.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.92042.log.gz 2026-03-23T19:58:52.270 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92086.log 2026-03-23T19:58:52.271 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92108.log 2026-03-23T19:58:52.271 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92063.log.gz 2026-03-23T19:58:52.271 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92086.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92086.log.gz 2026-03-23T19:58:52.272 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92130.log 2026-03-23T19:58:52.272 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92150.log 2026-03-23T19:58:52.272 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92108.log.gz 2026-03-23T19:58:52.273 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92167.log 2026-03-23T19:58:52.273 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92130.log.gz 2026-03-23T19:58:52.273 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92150.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92150.log.gz 2026-03-23T19:58:52.274 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92187.log 2026-03-23T19:58:52.274 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92207.log 2026-03-23T19:58:52.274 INFO:teuthology.orchestra.run.vm08.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92167.log.gz 2026-03-23T19:58:52.275 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92187.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92262.log 2026-03-23T19:58:52.275 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92187.log.gz 2026-03-23T19:58:52.276 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92283.log 2026-03-23T19:58:52.279 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92207.log.gz 2026-03-23T19:58:52.280 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92262.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92262.log.gz 2026-03-23T19:58:52.280 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92311.log 2026-03-23T19:58:52.281 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92283.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92283.log.gz 2026-03-23T19:58:52.281 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92332.log 2026-03-23T19:58:52.281 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92353.log 2026-03-23T19:58:52.285 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92311.log.gz 2026-03-23T19:58:52.285 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92332.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92376.log 2026-03-23T19:58:52.285 INFO:teuthology.orchestra.run.vm08.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92332.log.gz 2026-03-23T19:58:52.286 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92353.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92398.log 2026-03-23T19:58:52.286 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92353.log.gz 2026-03-23T19:58:52.287 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92418.log 2026-03-23T19:58:52.287 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.92376.log.gz 2026-03-23T19:58:52.288 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92398.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92398.log.gz 2026-03-23T19:58:52.288 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92435.log 2026-03-23T19:58:52.288 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92418.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92418.log.gz 2026-03-23T19:58:52.288 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92455.log 2026-03-23T19:58:52.289 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92475.log 2026-03-23T19:58:52.289 INFO:teuthology.orchestra.run.vm08.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92435.log.gz 2026-03-23T19:58:52.290 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92531.log 2026-03-23T19:58:52.290 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92551.log 2026-03-23T19:58:52.290 INFO:teuthology.orchestra.run.vm08.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92455.log.gz 2026-03-23T19:58:52.295 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92475.log.gz 2026-03-23T19:58:52.295 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92531.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92531.log.gz 2026-03-23T19:58:52.296 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92572.log 2026-03-23T19:58:52.296 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92551.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92551.log.gz 2026-03-23T19:58:52.297 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92599.log 2026-03-23T19:58:52.297 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92572.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92572.log.gz 2026-03-23T19:58:52.297 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92619.log 2026-03-23T19:58:52.298 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92599.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92599.log.gz 2026-03-23T19:58:52.298 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92640.log 2026-03-23T19:58:52.300 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92661.log 2026-03-23T19:58:52.302 INFO:teuthology.orchestra.run.vm08.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92619.log.gz 2026-03-23T19:58:52.303 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92640.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92640.log.gz 2026-03-23T19:58:52.303 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92684.log 2026-03-23T19:58:52.304 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92661.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92661.log.gz 2026-03-23T19:58:52.304 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92706.log 2026-03-23T19:58:52.305 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92684.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.92684.log.gz 2026-03-23T19:58:52.305 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92726.log 2026-03-23T19:58:52.306 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92706.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92706.log.gz 2026-03-23T19:58:52.308 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92743.log 2026-03-23T19:58:52.308 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92726.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92726.log.gz 2026-03-23T19:58:52.309 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92763.log 2026-03-23T19:58:52.309 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92743.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92743.log.gz 2026-03-23T19:58:52.310 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92783.log 2026-03-23T19:58:52.310 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92763.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92763.log.gz 2026-03-23T19:58:52.311 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92804.log 2026-03-23T19:58:52.311 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92783.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92824.log 2026-03-23T19:58:52.315 INFO:teuthology.orchestra.run.vm08.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92783.log.gz 2026-03-23T19:58:52.316 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92804.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92804.log.gz 2026-03-23T19:58:52.316 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92844.log 2026-03-23T19:58:52.317 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92824.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92824.log.gz 2026-03-23T19:58:52.317 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92864.log 2026-03-23T19:58:52.318 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92844.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92844.log.gz 2026-03-23T19:58:52.318 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92884.log 2026-03-23T19:58:52.319 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92864.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92864.log.gz 2026-03-23T19:58:52.319 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92906.log 2026-03-23T19:58:52.324 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92884.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92884.log.gz 2026-03-23T19:58:52.324 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92927.log 2026-03-23T19:58:52.324 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92906.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.92906.log.gz 2026-03-23T19:58:52.325 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92950.log 2026-03-23T19:58:52.325 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92927.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92927.log.gz 2026-03-23T19:58:52.325 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92972.log 2026-03-23T19:58:52.326 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92950.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92950.log.gz 2026-03-23T19:58:52.326 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92992.log 2026-03-23T19:58:52.327 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.92972.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92972.log.gz 2026-03-23T19:58:52.327 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93009.log 2026-03-23T19:58:52.328 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.92992.log: 42.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92992.log.gz 2026-03-23T19:58:52.328 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93029.log 2026-03-23T19:58:52.329 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93009.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93009.log.gz 2026-03-23T19:58:52.329 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93049.log 2026-03-23T19:58:52.330 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93029.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93029.log.gz 2026-03-23T19:58:52.330 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93069.log 2026-03-23T19:58:52.330 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93089.log 2026-03-23T19:58:52.331 INFO:teuthology.orchestra.run.vm08.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93049.log.gz 2026-03-23T19:58:52.331 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93069.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93069.log.gz 2026-03-23T19:58:52.332 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93109.log 2026-03-23T19:58:52.332 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93089.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93089.log.gz 2026-03-23T19:58:52.332 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93130.log 2026-03-23T19:58:52.333 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93109.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93150.log 2026-03-23T19:58:52.333 INFO:teuthology.orchestra.run.vm08.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93109.log.gz 2026-03-23T19:58:52.334 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93171.log 2026-03-23T19:58:52.334 INFO:teuthology.orchestra.run.vm08.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93130.log.gz 2026-03-23T19:58:52.336 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93150.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93150.log.gz 2026-03-23T19:58:52.336 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93191.log 2026-03-23T19:58:52.337 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93171.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93171.log.gz 2026-03-23T19:58:52.337 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93212.log 2026-03-23T19:58:52.338 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93191.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93191.log.gz 2026-03-23T19:58:52.338 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93232.log 2026-03-23T19:58:52.339 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93212.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93212.log.gz 2026-03-23T19:58:52.339 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93254.log 2026-03-23T19:58:52.340 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93277.log 2026-03-23T19:58:52.341 INFO:teuthology.orchestra.run.vm08.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93232.log.gz 2026-03-23T19:58:52.341 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93254.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93254.log.gz 2026-03-23T19:58:52.342 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93299.log 2026-03-23T19:58:52.342 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93277.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93277.log.gz 2026-03-23T19:58:52.342 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93322.log 2026-03-23T19:58:52.343 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93299.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93299.log.gz 2026-03-23T19:58:52.343 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93344.log 2026-03-23T19:58:52.344 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93322.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93322.log.gz 2026-03-23T19:58:52.344 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93367.log 2026-03-23T19:58:52.344 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93344.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93344.log.gz 2026-03-23T19:58:52.345 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93389.log 2026-03-23T19:58:52.345 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93367.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93367.log.gz 2026-03-23T19:58:52.345 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93412.log 2026-03-23T19:58:52.346 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93389.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93389.log.gz 2026-03-23T19:58:52.346 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93436.log 2026-03-23T19:58:52.347 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93412.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93412.log.gz 2026-03-23T19:58:52.347 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93456.log 2026-03-23T19:58:52.348 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93436.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93436.log.gz 2026-03-23T19:58:52.348 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93476.log 2026-03-23T19:58:52.349 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93456.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93456.log.gz 2026-03-23T19:58:52.349 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93498.log 2026-03-23T19:58:52.350 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93476.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93476.log.gz 2026-03-23T19:58:52.350 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93519.log 2026-03-23T19:58:52.350 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93540.log 2026-03-23T19:58:52.351 INFO:teuthology.orchestra.run.vm08.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93498.log.gz 2026-03-23T19:58:52.351 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93561.log 2026-03-23T19:58:52.352 INFO:teuthology.orchestra.run.vm08.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93519.log.gz 2026-03-23T19:58:52.352 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93540.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93540.log.gz 2026-03-23T19:58:52.352 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93582.log 2026-03-23T19:58:52.353 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93603.log 2026-03-23T19:58:52.353 INFO:teuthology.orchestra.run.vm08.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93561.log.gz 2026-03-23T19:58:52.354 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93582.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93582.log.gz 2026-03-23T19:58:52.354 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93624.log 2026-03-23T19:58:52.355 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93603.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93603.log.gz 2026-03-23T19:58:52.355 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93645.log 2026-03-23T19:58:52.356 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93624.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93624.log.gz 2026-03-23T19:58:52.356 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93666.log 2026-03-23T19:58:52.357 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93687.log 2026-03-23T19:58:52.357 INFO:teuthology.orchestra.run.vm08.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93645.log.gz 2026-03-23T19:58:52.358 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93666.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93666.log.gz 2026-03-23T19:58:52.358 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93708.log 2026-03-23T19:58:52.358 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93687.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93687.log.gz 2026-03-23T19:58:52.359 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93729.log 2026-03-23T19:58:52.359 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93708.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93708.log.gz 2026-03-23T19:58:52.360 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93750.log 2026-03-23T19:58:52.360 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93771.log 2026-03-23T19:58:52.360 INFO:teuthology.orchestra.run.vm08.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93729.log.gz 2026-03-23T19:58:52.361 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93750.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93750.log.gz 2026-03-23T19:58:52.361 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93792.log 2026-03-23T19:58:52.362 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93771.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93771.log.gz 2026-03-23T19:58:52.362 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93813.log 2026-03-23T19:58:52.363 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93792.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93792.log.gz 2026-03-23T19:58:52.366 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93834.log 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93813.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93813.log.gz 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93855.log 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93834.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93834.log.gz 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93876.log 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93855.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93855.log.gz 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93897.log 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93876.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93876.log.gz 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93917.log 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93897.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93897.log.gz 2026-03-23T19:58:52.367 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93938.log 2026-03-23T19:58:52.368 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93917.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93917.log.gz 2026-03-23T19:58:52.369 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93959.log 2026-03-23T19:58:52.369 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93938.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.93938.log.gz 2026-03-23T19:58:52.369 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93979.log 2026-03-23T19:58:52.370 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.93959.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93959.log.gz 2026-03-23T19:58:52.370 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93999.log 2026-03-23T19:58:52.371 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93979.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93979.log.gz 2026-03-23T19:58:52.371 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94020.log 2026-03-23T19:58:52.372 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.93999.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93999.log.gz 2026-03-23T19:58:52.372 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94040.log 2026-03-23T19:58:52.373 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94020.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.94020.log.gz 2026-03-23T19:58:52.373 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94062.log 2026-03-23T19:58:52.375 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94040.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94040.log.gz 2026-03-23T19:58:52.375 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94082.log 2026-03-23T19:58:52.376 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94062.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94062.log.gz 2026-03-23T19:58:52.376 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94106.log 2026-03-23T19:58:52.377 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94082.log: 91.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94082.log.gz 2026-03-23T19:58:52.378 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94123.log 2026-03-23T19:58:52.378 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94106.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94106.log.gz 2026-03-23T19:58:52.378 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94143.log 2026-03-23T19:58:52.379 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94123.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94123.log.gz 2026-03-23T19:58:52.379 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94163.log 2026-03-23T19:58:52.380 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94143.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94143.log.gz 2026-03-23T19:58:52.380 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94234.log 2026-03-23T19:58:52.384 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94163.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94163.log.gz 2026-03-23T19:58:52.385 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94260.log 2026-03-23T19:58:52.385 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94234.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94234.log.gz 2026-03-23T19:58:52.386 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94284.log 2026-03-23T19:58:52.386 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94260.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94260.log.gz 2026-03-23T19:58:52.387 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94308.log 2026-03-23T19:58:52.387 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.94284.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.94284.log.gz 2026-03-23T19:58:52.387 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94332.log 2026-03-23T19:58:52.388 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.94308.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.94308.log.gz 2026-03-23T19:58:52.388 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94356.log 2026-03-23T19:58:52.389 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.94332.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.94332.log.gz 2026-03-23T19:58:52.389 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94383.log 2026-03-23T19:58:52.389 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.94356.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94356.log.gz 2026-03-23T19:58:52.390 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94410.log 2026-03-23T19:58:52.390 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.94383.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.94383.log.gz 2026-03-23T19:58:52.390 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94437.log 2026-03-23T19:58:52.391 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.94410.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.94410.log.gz 2026-03-23T19:58:52.391 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94564.log 2026-03-23T19:58:52.392 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.94437.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.94437.log.gz 2026-03-23T19:58:52.392 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94585.log 2026-03-23T19:58:52.393 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94564.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.94606.log 2026-03-23T19:58:52.397 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94585.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94564.log.gz 2026-03-23T19:58:52.398 INFO:teuthology.orchestra.run.vm08.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94585.log.gz 2026-03-23T19:58:52.398 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.94615.log 2026-03-23T19:58:52.399 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.94606.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.94606.log.gz 2026-03-23T19:58:52.399 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.94625.log 2026-03-23T19:58:52.399 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.94615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.94650.log 2026-03-23T19:58:52.401 INFO:teuthology.orchestra.run.vm08.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.94615.log.gz 2026-03-23T19:58:52.402 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.94625.log: 18.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.94625.log.gz 2026-03-23T19:58:52.402 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.94658.log 2026-03-23T19:58:52.406 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.94650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.94685.log 2026-03-23T19:58:52.408 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.94658.log: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.94658.log.gz 2026-03-23T19:58:52.410 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.94690.log 2026-03-23T19:58:52.414 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.94685.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.94650.log.gz 2026-03-23T19:58:52.415 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.94745.log 2026-03-23T19:58:52.417 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.94690.log: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.94690.log.gz 2026-03-23T19:58:52.418 INFO:teuthology.orchestra.run.vm08.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.94685.log.gz 2026-03-23T19:58:52.419 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94780.log 2026-03-23T19:58:52.423 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.94745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95010.log 2026-03-23T19:58:52.427 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.94780.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94780.log.gz 2026-03-23T19:58:52.428 INFO:teuthology.orchestra.run.vm08.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.94745.log.gz 2026-03-23T19:58:52.429 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95036.log 2026-03-23T19:58:52.429 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95010.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.95010.log.gz 2026-03-23T19:58:52.429 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95060.log 2026-03-23T19:58:52.430 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95036.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.95036.log.gz 2026-03-23T19:58:52.430 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95084.log 2026-03-23T19:58:52.430 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95060.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95060.log.gz 2026-03-23T19:58:52.431 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95124.log 2026-03-23T19:58:52.431 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95084.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.95084.log.gz 2026-03-23T19:58:52.431 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95148.log 2026-03-23T19:58:52.432 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95124.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95124.log.gz 2026-03-23T19:58:52.432 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95175.log 2026-03-23T19:58:52.433 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95148.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.95148.log.gz 2026-03-23T19:58:52.433 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95202.log 2026-03-23T19:58:52.433 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95175.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95229.log 2026-03-23T19:58:52.434 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.95175.log.gz 2026-03-23T19:58:52.434 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95202.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95262.log 2026-03-23T19:58:52.434 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.95202.log.gz 2026-03-23T19:58:52.435 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95282.log 2026-03-23T19:58:52.435 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.95229.log.gz 2026-03-23T19:58:52.436 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95262.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95262.log.gz 2026-03-23T19:58:52.437 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95304.log 2026-03-23T19:58:52.438 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95282.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.95282.log.gz 2026-03-23T19:58:52.438 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95375.log 2026-03-23T19:58:52.439 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95304.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95402.log 2026-03-23T19:58:52.439 INFO:teuthology.orchestra.run.vm08.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.95304.log.gz 2026-03-23T19:58:52.440 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95375.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.95375.log.gz 2026-03-23T19:58:52.440 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95429.log 2026-03-23T19:58:52.441 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95402.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.95402.log.gz 2026-03-23T19:58:52.441 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95456.log 2026-03-23T19:58:52.442 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95429.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.95429.log.gz 2026-03-23T19:58:52.442 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95483.log 2026-03-23T19:58:52.443 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95456.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.admin.95456.log.gz 2026-03-23T19:58:52.443 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95503.log 2026-03-23T19:58:52.443 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95483.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.95483.log.gz 2026-03-23T19:58:52.444 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95527.log 2026-03-23T19:58:52.444 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95544.log 2026-03-23T19:58:52.445 INFO:teuthology.orchestra.run.vm08.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.95503.log.gz 2026-03-23T19:58:52.446 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95564.log 2026-03-23T19:58:52.446 INFO:teuthology.orchestra.run.vm08.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.95527.log.gz 2026-03-23T19:58:52.447 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95544.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95544.log.gz 2026-03-23T19:58:52.447 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95584.log 2026-03-23T19:58:52.447 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95564.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95564.log.gz 2026-03-23T19:58:52.448 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95657.log 2026-03-23T19:58:52.448 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95683.log 2026-03-23T19:58:52.452 INFO:teuthology.orchestra.run.vm08.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95584.log.gz 2026-03-23T19:58:52.452 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95657.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95657.log.gz 2026-03-23T19:58:52.453 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95707.log 2026-03-23T19:58:52.453 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95683.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95683.log.gz 2026-03-23T19:58:52.453 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95731.log 2026-03-23T19:58:52.454 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95707.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.95707.log.gz 2026-03-23T19:58:52.454 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95755.log 2026-03-23T19:58:52.454 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95731.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95731.log.gz 2026-03-23T19:58:52.455 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95782.log 2026-03-23T19:58:52.455 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95755.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.95755.log.gz 2026-03-23T19:58:52.455 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95809.log 2026-03-23T19:58:52.456 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.95782.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.admin.95782.log.gz 2026-03-23T19:58:52.456 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95836.log 2026-03-23T19:58:52.457 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95809.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.95809.log.gz 2026-03-23T19:58:52.457 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95862.log 2026-03-23T19:58:52.457 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95883.log 2026-03-23T19:58:52.458 INFO:teuthology.orchestra.run.vm08.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.95836.log.gz 2026-03-23T19:58:52.461 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95904.log 2026-03-23T19:58:52.461 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.95862.log.gz 2026-03-23T19:58:52.462 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.95883.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95925.log 2026-03-23T19:58:52.463 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95883.log.gz 2026-03-23T19:58:52.463 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95904.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95904.log.gz 2026-03-23T19:58:52.464 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95946.log 2026-03-23T19:58:52.464 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.95925.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.95925.log.gz 2026-03-23T19:58:52.464 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95973.log 2026-03-23T19:58:52.465 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95946.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.95946.log.gz 2026-03-23T19:58:52.465 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96000.log 2026-03-23T19:58:52.466 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.95973.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.95973.log.gz 2026-03-23T19:58:52.466 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96023.log 2026-03-23T19:58:52.467 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96000.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.96000.log.gz 2026-03-23T19:58:52.467 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96050.log 2026-03-23T19:58:52.468 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96023.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.96023.log.gz 2026-03-23T19:58:52.468 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96076.log 2026-03-23T19:58:52.469 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96050.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.96050.log.gz 2026-03-23T19:58:52.469 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96117.log 2026-03-23T19:58:52.470 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96076.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.96076.log.gz 2026-03-23T19:58:52.470 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96139.log 2026-03-23T19:58:52.470 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96117.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.96117.log.gz 2026-03-23T19:58:52.471 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96166.log 2026-03-23T19:58:52.471 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96139.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.96139.log.gz 2026-03-23T19:58:52.471 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96193.log 2026-03-23T19:58:52.472 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96166.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.96166.log.gz 2026-03-23T19:58:52.472 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96220.log 2026-03-23T19:58:52.473 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.96193.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.96193.log.gz 2026-03-23T19:58:52.473 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96253.log 2026-03-23T19:58:52.474 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.96220.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.96220.log.gz 2026-03-23T19:58:52.474 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96273.log 2026-03-23T19:58:52.475 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96253.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.96253.log.gz 2026-03-23T19:58:52.475 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96296.log 2026-03-23T19:58:52.476 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96273.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.96273.log.gz 2026-03-23T19:58:52.477 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96316.log 2026-03-23T19:58:52.477 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96340.log 2026-03-23T19:58:52.479 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96316.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.96316.log.gz 2026-03-23T19:58:52.479 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96357.log 2026-03-23T19:58:52.479 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96340.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96340.log.gz 2026-03-23T19:58:52.479 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96377.log 2026-03-23T19:58:52.480 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96357.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.96357.log.gz 2026-03-23T19:58:52.480 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96397.log 2026-03-23T19:58:52.481 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96377.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96377.log.gz 2026-03-23T19:58:52.481 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96468.log 2026-03-23T19:58:52.483 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96296.log: /var/log/ceph/cluster2-client.mirror.96397.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.96296.log.gz 2026-03-23T19:58:52.485 INFO:teuthology.orchestra.run.vm08.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96397.log.gz 2026-03-23T19:58:52.485 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96494.log 2026-03-23T19:58:52.486 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96468.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.96468.log.gz 2026-03-23T19:58:52.486 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96518.log 2026-03-23T19:58:52.487 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96494.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.96494.log.gz 2026-03-23T19:58:52.487 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96542.log 2026-03-23T19:58:52.487 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96518.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.96518.log.gz 2026-03-23T19:58:52.488 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96569.log 2026-03-23T19:58:52.490 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96542.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.96542.log.gz 2026-03-23T19:58:52.490 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96596.log 2026-03-23T19:58:52.490 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.96569.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.96569.log.gz 2026-03-23T19:58:52.490 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96623.log 2026-03-23T19:58:52.490 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.96596.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.96596.log.gz 2026-03-23T19:58:52.491 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96750.log 2026-03-23T19:58:52.491 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.96623.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.96623.log.gz 2026-03-23T19:58:52.492 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96771.log 2026-03-23T19:58:52.492 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96750.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96792.log 2026-03-23T19:58:52.496 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96750.log.gz 2026-03-23T19:58:52.497 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.96771.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96771.log.gz 2026-03-23T19:58:52.497 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96813.log 2026-03-23T19:58:52.497 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96792.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.96792.log.gz 2026-03-23T19:58:52.497 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.96834.log 2026-03-23T19:58:52.498 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.96813.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.96813.log.gz 2026-03-23T19:58:52.498 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.96843.log 2026-03-23T19:58:52.499 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.96834.log: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.96834.log.gz 2026-03-23T19:58:52.499 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.96848.log 2026-03-23T19:58:52.500 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.96843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.96887.log 2026-03-23T19:58:52.502 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.96848.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.96843.log.gz 2026-03-23T19:58:52.503 INFO:teuthology.orchestra.run.vm08.stderr: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.96848.log.gz 2026-03-23T19:58:52.503 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.96898.log 2026-03-23T19:58:52.507 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.96887.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.96887.log.gz 2026-03-23T19:58:52.508 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.96963.log 2026-03-23T19:58:52.508 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.96898.log: 17.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.96898.log.gz 2026-03-23T19:58:52.508 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.96968.log 2026-03-23T19:58:52.509 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.96963.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.97049.log 2026-03-23T19:58:52.511 INFO:teuthology.orchestra.run.vm08.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.96963.log.gz 2026-03-23T19:58:52.512 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.96968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97211.log 2026-03-23T19:58:52.512 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.97049.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.96968.log.gz 2026-03-23T19:58:52.514 INFO:teuthology.orchestra.run.vm08.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.97049.log.gz 2026-03-23T19:58:52.514 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97238.log 2026-03-23T19:58:52.515 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.97211.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.97211.log.gz 2026-03-23T19:58:52.516 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97265.log 2026-03-23T19:58:52.516 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.97238.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.97238.log.gz 2026-03-23T19:58:52.517 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97303.log 2026-03-23T19:58:52.517 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97265.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.97265.log.gz 2026-03-23T19:58:52.518 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97325.log 2026-03-23T19:58:52.518 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97303.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.97303.log.gz 2026-03-23T19:58:52.518 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97348.log 2026-03-23T19:58:52.519 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97325.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.97325.log.gz 2026-03-23T19:58:52.519 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97375.log 2026-03-23T19:58:52.520 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97348.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.97348.log.gz 2026-03-23T19:58:52.520 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97401.log 2026-03-23T19:58:52.520 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97375.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.97375.log.gz 2026-03-23T19:58:52.521 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97454.log 2026-03-23T19:58:52.522 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.97401.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97401.log.gz 2026-03-23T19:58:52.522 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97481.log 2026-03-23T19:58:52.522 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97454.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.97454.log.gz 2026-03-23T19:58:52.522 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97508.log 2026-03-23T19:58:52.523 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.97481.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.admin.97481.log.gz 2026-03-23T19:58:52.523 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97535.log 2026-03-23T19:58:52.524 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.97508.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.97508.log.gz 2026-03-23T19:58:52.524 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97568.log 2026-03-23T19:58:52.525 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.97535.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.97535.log.gz 2026-03-23T19:58:52.525 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97588.log 2026-03-23T19:58:52.526 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.97568.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97568.log.gz 2026-03-23T19:58:52.526 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97611.log 2026-03-23T19:58:52.527 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.97588.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97588.log.gz 2026-03-23T19:58:52.527 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97631.log 2026-03-23T19:58:52.528 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.97611.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97611.log.gz 2026-03-23T19:58:52.528 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97857.log 2026-03-23T19:58:52.529 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.97631.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97631.log.gz 2026-03-23T19:58:52.530 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97878.log 2026-03-23T19:58:52.530 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.97857.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97857.log.gz 2026-03-23T19:58:52.530 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.97900.log 2026-03-23T19:58:52.531 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.97878.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.97878.log.gz 2026-03-23T19:58:52.531 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.97909.log 2026-03-23T19:58:52.531 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.97900.log: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.97900.log.gz 2026-03-23T19:58:52.532 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.97914.log 2026-03-23T19:58:52.537 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.97909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.97944.log 2026-03-23T19:58:52.544 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.97914.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.97956.log 2026-03-23T19:58:52.544 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.97944.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.97914.log.gz 2026-03-23T19:58:52.550 INFO:teuthology.orchestra.run.vm08.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.97944.log.gz 2026-03-23T19:58:52.555 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.97997.log 2026-03-23T19:58:52.556 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.98008.log 2026-03-23T19:58:52.558 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.97956.log: 17.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.97956.log.gz 2026-03-23T19:58:52.560 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.98080.log 2026-03-23T19:58:52.560 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.97997.log: /var/log/ceph/cluster1-client.mirror.3.98008.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.98008.log.gz 2026-03-23T19:58:52.562 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98109.log 2026-03-23T19:58:52.567 INFO:teuthology.orchestra.run.vm08.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.97997.log.gz 2026-03-23T19:58:52.569 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98299.log 2026-03-23T19:58:52.570 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98109.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98109.log.gz 2026-03-23T19:58:52.577 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.98080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98323.log 2026-03-23T19:58:52.577 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98299.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98299.log.gz 2026-03-23T19:58:52.579 INFO:teuthology.orchestra.run.vm08.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.97909.log.gz 2026-03-23T19:58:52.579 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98346.log 2026-03-23T19:58:52.580 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98323.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98323.log.gz 2026-03-23T19:58:52.580 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98373.log 2026-03-23T19:58:52.580 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98346.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.98346.log.gz 2026-03-23T19:58:52.580 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98394.log 2026-03-23T19:58:52.581 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98373.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98373.log.gz 2026-03-23T19:58:52.581 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98414.log 2026-03-23T19:58:52.582 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98394.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98394.log.gz 2026-03-23T19:58:52.582 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98435.log 2026-03-23T19:58:52.582 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98414.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98414.log.gz 2026-03-23T19:58:52.583 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98458.log 2026-03-23T19:58:52.583 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98435.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.98435.log.gz 2026-03-23T19:58:52.583 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98481.log 2026-03-23T19:58:52.584 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98458.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98458.log.gz 2026-03-23T19:58:52.584 INFO:teuthology.orchestra.run.vm08.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.98080.log.gz 2026-03-23T19:58:52.584 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98509.log 2026-03-23T19:58:52.584 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98481.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98481.log.gz 2026-03-23T19:58:52.585 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98548.log 2026-03-23T19:58:52.585 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98565.log 2026-03-23T19:58:52.585 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98509.log.gz 2026-03-23T19:58:52.586 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98548.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.98548.log.gz 2026-03-23T19:58:52.586 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98585.log 2026-03-23T19:58:52.586 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98605.log 2026-03-23T19:58:52.587 INFO:teuthology.orchestra.run.vm08.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98565.log.gz 2026-03-23T19:58:52.587 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98622.log 2026-03-23T19:58:52.587 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.98585.log.gz 2026-03-23T19:58:52.588 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98605.log: 79.1% -- replaced with /var/log/ceph/cluster2-client.mirror.98605.log.gz 2026-03-23T19:58:52.588 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98642.log 2026-03-23T19:58:52.589 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98622.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98622.log.gz 2026-03-23T19:58:52.589 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98686.log 2026-03-23T19:58:52.590 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98707.log 2026-03-23T19:58:52.590 INFO:teuthology.orchestra.run.vm08.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.98642.log.gz 2026-03-23T19:58:52.590 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98728.log 2026-03-23T19:58:52.594 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.98686.log.gz 2026-03-23T19:58:52.594 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98754.log 2026-03-23T19:58:52.597 INFO:teuthology.orchestra.run.vm08.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98707.log.gz 2026-03-23T19:58:52.598 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98728.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.98728.log.gz 2026-03-23T19:58:52.598 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98778.log 2026-03-23T19:58:52.598 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98754.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98754.log.gz 2026-03-23T19:58:52.598 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98802.log 2026-03-23T19:58:52.599 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98778.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98778.log.gz 2026-03-23T19:58:52.599 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98825.log 2026-03-23T19:58:52.599 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98802.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98802.log.gz 2026-03-23T19:58:52.600 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98851.log 2026-03-23T19:58:52.600 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98825.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.98825.log.gz 2026-03-23T19:58:52.601 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98875.log 2026-03-23T19:58:52.601 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.98851.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98851.log.gz 2026-03-23T19:58:52.601 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98899.log 2026-03-23T19:58:52.602 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98875.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98875.log.gz 2026-03-23T19:58:52.602 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98923.log 2026-03-23T19:58:52.602 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.98899.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98899.log.gz 2026-03-23T19:58:52.603 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98950.log 2026-03-23T19:58:52.603 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.98923.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.98923.log.gz 2026-03-23T19:58:52.603 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98977.log 2026-03-23T19:58:52.604 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.98950.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.98950.log.gz 2026-03-23T19:58:52.604 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.99004.log 2026-03-23T19:58:52.604 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.98977.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.98977.log.gz 2026-03-23T19:58:52.605 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99037.log 2026-03-23T19:58:52.605 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.99004.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.99004.log.gz 2026-03-23T19:58:52.605 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99057.log 2026-03-23T19:58:52.606 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99037.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99037.log.gz 2026-03-23T19:58:52.606 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99086.log 2026-03-23T19:58:52.608 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99057.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.99057.log.gz 2026-03-23T19:58:52.608 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99106.log 2026-03-23T19:58:52.609 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99086.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99086.log.gz 2026-03-23T19:58:52.609 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99132.log 2026-03-23T19:58:52.610 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99106.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.99106.log.gz 2026-03-23T19:58:52.610 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99152.log 2026-03-23T19:58:52.611 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99132.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.99132.log.gz 2026-03-23T19:58:52.611 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99173.log 2026-03-23T19:58:52.612 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99152.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99152.log.gz 2026-03-23T19:58:52.613 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99194.log 2026-03-23T19:58:52.613 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99173.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.99173.log.gz 2026-03-23T19:58:52.613 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99217.log 2026-03-23T19:58:52.614 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99194.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99194.log.gz 2026-03-23T19:58:52.614 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99239.log 2026-03-23T19:58:52.614 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99217.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99217.log.gz 2026-03-23T19:58:52.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99262.log 2026-03-23T19:58:52.615 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99239.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99239.log.gz 2026-03-23T19:58:52.615 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99284.log 2026-03-23T19:58:52.616 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99262.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99262.log.gz 2026-03-23T19:58:52.616 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99304.log 2026-03-23T19:58:52.616 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99284.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.99284.log.gz 2026-03-23T19:58:52.617 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99331.log 2026-03-23T19:58:52.618 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99304.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99304.log.gz 2026-03-23T19:58:52.618 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99348.log 2026-03-23T19:58:52.619 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99331.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.99331.log.gz 2026-03-23T19:58:52.619 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99365.log 2026-03-23T19:58:52.619 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99348.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99348.log.gz 2026-03-23T19:58:52.619 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99382.log 2026-03-23T19:58:52.620 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99365.log: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99365.log.gz 2026-03-23T19:58:52.620 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99399.log 2026-03-23T19:58:52.620 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99382.log: 69.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99382.log.gz 2026-03-23T19:58:52.621 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99416.log 2026-03-23T19:58:52.621 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99399.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99399.log.gz 2026-03-23T19:58:52.621 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99436.log 2026-03-23T19:58:52.622 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99416.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99416.log.gz 2026-03-23T19:58:52.622 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99522.log 2026-03-23T19:58:52.623 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99436.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.99436.log.gz 2026-03-23T19:58:52.624 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99543.log 2026-03-23T19:58:52.624 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99569.log 2026-03-23T19:58:52.628 INFO:teuthology.orchestra.run.vm08.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.99522.log.gz 2026-03-23T19:58:52.629 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99543.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99593.log 2026-03-23T19:58:52.629 INFO:teuthology.orchestra.run.vm08.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99543.log.gz 2026-03-23T19:58:52.630 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99569.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99569.log.gz 2026-03-23T19:58:52.630 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.99617.log 2026-03-23T19:58:52.631 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99593.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.99593.log.gz 2026-03-23T19:58:52.631 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.99644.log 2026-03-23T19:58:52.632 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.99617.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.99617.log.gz 2026-03-23T19:58:52.632 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99677.log 2026-03-23T19:58:52.633 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.99644.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.99644.log.gz 2026-03-23T19:58:52.633 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99697.log 2026-03-23T19:58:52.634 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99677.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99677.log.gz 2026-03-23T19:58:52.635 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99723.log 2026-03-23T19:58:52.636 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99697.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.99697.log.gz 2026-03-23T19:58:52.637 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99743.log 2026-03-23T19:58:52.637 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99723.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99723.log.gz 2026-03-23T19:58:52.638 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99765.log 2026-03-23T19:58:52.639 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99743.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99743.log.gz 2026-03-23T19:58:52.640 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99788.log 2026-03-23T19:58:52.640 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99765.log: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.99765.log.gz 2026-03-23T19:58:52.640 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99809.log 2026-03-23T19:58:52.641 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99788.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99788.log.gz 2026-03-23T19:58:52.641 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99826.log 2026-03-23T19:58:52.642 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99809.log: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99809.log.gz 2026-03-23T19:58:52.642 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99843.log 2026-03-23T19:58:52.643 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99826.log: 56.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99826.log.gz 2026-03-23T19:58:52.643 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99860.log 2026-03-23T19:58:52.643 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99843.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99843.log.gz 2026-03-23T19:58:52.644 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99877.log 2026-03-23T19:58:52.644 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99860.log: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.99860.log.gz 2026-03-23T19:58:52.644 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99894.log 2026-03-23T19:58:52.645 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.99877.log: 64.6% -- replaced with /var/log/ceph/cluster1-client.mirror.99877.log.gz 2026-03-23T19:58:52.645 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99911.log 2026-03-23T19:58:52.646 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99894.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99894.log.gz 2026-03-23T19:58:52.646 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99931.log 2026-03-23T19:58:52.647 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99911.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99911.log.gz 2026-03-23T19:58:52.647 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99999.log 2026-03-23T19:58:52.648 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99931.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.99931.log.gz 2026-03-23T19:58:52.648 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100020.log 2026-03-23T19:58:52.649 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.99999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100046.log 2026-03-23T19:58:52.653 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.99999.log.gz 2026-03-23T19:58:52.654 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100020.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.100020.log.gz 2026-03-23T19:58:52.654 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100070.log 2026-03-23T19:58:52.655 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100046.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.100046.log.gz 2026-03-23T19:58:52.655 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100094.log 2026-03-23T19:58:52.656 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100070.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100070.log.gz 2026-03-23T19:58:52.656 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100118.log 2026-03-23T19:58:52.657 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100094.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.100094.log.gz 2026-03-23T19:58:52.657 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.100142.log 2026-03-23T19:58:52.658 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.100169.log 2026-03-23T19:58:52.658 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.100118.log.gz 2026-03-23T19:58:52.659 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.100142.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.100142.log.gz 2026-03-23T19:58:52.659 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100202.log 2026-03-23T19:58:52.659 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.100169.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.100169.log.gz 2026-03-23T19:58:52.660 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100222.log 2026-03-23T19:58:52.661 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100202.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100202.log.gz 2026-03-23T19:58:52.662 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100245.log 2026-03-23T19:58:52.663 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100222.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.100222.log.gz 2026-03-23T19:58:52.663 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100265.log 2026-03-23T19:58:52.664 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100245.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100245.log.gz 2026-03-23T19:58:52.664 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100287.log 2026-03-23T19:58:52.666 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100265.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.100265.log.gz 2026-03-23T19:58:52.666 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100310.log 2026-03-23T19:58:52.667 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100287.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100287.log.gz 2026-03-23T19:58:52.667 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100331.log 2026-03-23T19:58:52.668 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100310.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100310.log.gz 2026-03-23T19:58:52.668 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100348.log 2026-03-23T19:58:52.668 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100331.log: 57.4% -- replaced with /var/log/ceph/cluster1-client.mirror.100331.log.gz 2026-03-23T19:58:52.669 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100368.log 2026-03-23T19:58:52.669 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100348.log: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.100348.log.gz 2026-03-23T19:58:52.669 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100385.log 2026-03-23T19:58:52.670 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100368.log: 79.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100368.log.gz 2026-03-23T19:58:52.670 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100405.log 2026-03-23T19:58:52.671 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100385.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.100385.log.gz 2026-03-23T19:58:52.671 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100426.log 2026-03-23T19:58:52.672 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100405.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.100405.log.gz 2026-03-23T19:58:52.672 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100478.log 2026-03-23T19:58:52.673 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100426.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100426.log.gz 2026-03-23T19:58:52.673 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100499.log 2026-03-23T19:58:52.674 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100478.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.100478.log.gz 2026-03-23T19:58:52.674 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100521.log 2026-03-23T19:58:52.679 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100499.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.100499.log.gz 2026-03-23T19:58:52.679 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100547.log 2026-03-23T19:58:52.680 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100521.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100521.log.gz 2026-03-23T19:58:52.680 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100571.log 2026-03-23T19:58:52.680 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100547.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.100547.log.gz 2026-03-23T19:58:52.681 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100595.log 2026-03-23T19:58:52.681 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100571.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100571.log.gz 2026-03-23T19:58:52.681 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100619.log 2026-03-23T19:58:52.682 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100595.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.100595.log.gz 2026-03-23T19:58:52.682 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.100643.log 2026-03-23T19:58:52.683 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100619.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.100619.log.gz 2026-03-23T19:58:52.683 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.100670.log 2026-03-23T19:58:52.684 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.100643.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.100643.log.gz 2026-03-23T19:58:52.684 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100703.log 2026-03-23T19:58:52.685 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.100670.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.100670.log.gz 2026-03-23T19:58:52.685 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100723.log 2026-03-23T19:58:52.686 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100703.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100703.log.gz 2026-03-23T19:58:52.687 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100746.log 2026-03-23T19:58:52.688 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100723.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.100723.log.gz 2026-03-23T19:58:52.688 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100766.log 2026-03-23T19:58:52.689 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100746.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100746.log.gz 2026-03-23T19:58:52.689 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100791.log 2026-03-23T19:58:52.691 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.100766.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100766.log.gz 2026-03-23T19:58:52.691 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100812.log 2026-03-23T19:58:52.692 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100791.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.100791.log.gz 2026-03-23T19:58:52.692 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100834.log 2026-03-23T19:58:52.693 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100812.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.100812.log.gz 2026-03-23T19:58:52.693 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100856.log 2026-03-23T19:58:52.694 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100834.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100834.log.gz 2026-03-23T19:58:52.694 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100879.log 2026-03-23T19:58:52.694 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100856.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.100856.log.gz 2026-03-23T19:58:52.695 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100902.log 2026-03-23T19:58:52.695 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100879.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100879.log.gz 2026-03-23T19:58:52.695 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100925.log 2026-03-23T19:58:52.696 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100902.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100902.log.gz 2026-03-23T19:58:52.696 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100948.log 2026-03-23T19:58:52.696 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100925.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100925.log.gz 2026-03-23T19:58:52.697 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.100982.log 2026-03-23T19:58:52.697 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.100948.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.100948.log.gz 2026-03-23T19:58:52.698 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.101009.log 2026-03-23T19:58:52.698 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.100982.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.100982.log.gz 2026-03-23T19:58:52.699 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101042.log 2026-03-23T19:58:52.699 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.101009.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.101009.log.gz 2026-03-23T19:58:52.700 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101062.log 2026-03-23T19:58:52.701 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.101042.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.101042.log.gz 2026-03-23T19:58:52.702 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101106.log 2026-03-23T19:58:52.703 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101062.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101062.log.gz 2026-03-23T19:58:52.703 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101127.log 2026-03-23T19:58:52.704 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101106.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101106.log.gz 2026-03-23T19:58:52.704 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101155.log 2026-03-23T19:58:52.705 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101127.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.101127.log.gz 2026-03-23T19:58:52.705 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101183.log 2026-03-23T19:58:52.705 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101155.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101155.log.gz 2026-03-23T19:58:52.706 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101206.log 2026-03-23T19:58:52.706 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101183.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101183.log.gz 2026-03-23T19:58:52.706 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.101240.log 2026-03-23T19:58:52.707 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101206.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101206.log.gz 2026-03-23T19:58:52.707 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.101267.log 2026-03-23T19:58:52.708 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.101240.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.101240.log.gz 2026-03-23T19:58:52.708 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101300.log 2026-03-23T19:58:52.709 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.101267.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.101267.log.gz 2026-03-23T19:58:52.709 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101320.log 2026-03-23T19:58:52.710 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.101300.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101300.log.gz 2026-03-23T19:58:52.710 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101447.log 2026-03-23T19:58:52.712 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101320.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101320.log.gz 2026-03-23T19:58:52.712 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101468.log 2026-03-23T19:58:52.712 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101447.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101447.log.gz 2026-03-23T19:58:52.713 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.101489.log 2026-03-23T19:58:52.713 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.101498.log 2026-03-23T19:58:52.713 INFO:teuthology.orchestra.run.vm08.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101468.log.gz 2026-03-23T19:58:52.714 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.101489.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.101489.log.gz 2026-03-23T19:58:52.714 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.101506.log 2026-03-23T19:58:52.717 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.101498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.101542.log 2026-03-23T19:58:52.720 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.101506.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.101506.log.gz 2026-03-23T19:58:52.730 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.101553.log 2026-03-23T19:58:52.740 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.101542.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.101542.log.gz 2026-03-23T19:58:52.742 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.101593.log 2026-03-23T19:58:52.742 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.101553.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.101553.log.gz 2026-03-23T19:58:52.748 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.101601.log 2026-03-23T19:58:52.756 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.101593.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.101593.log.gz 2026-03-23T19:58:52.758 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.101653.log 2026-03-23T19:58:52.760 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.101601.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.101601.log.gz 2026-03-23T19:58:52.767 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101722.log 2026-03-23T19:58:52.772 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.101653.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.101653.log.gz 2026-03-23T19:58:52.778 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101890.log 2026-03-23T19:58:52.779 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101722.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.101722.log.gz 2026-03-23T19:58:52.783 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101913.log 2026-03-23T19:58:52.786 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101890.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101890.log.gz 2026-03-23T19:58:52.792 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101948.log 2026-03-23T19:58:52.793 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101913.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101913.log.gz 2026-03-23T19:58:52.798 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101970.log 2026-03-23T19:58:52.800 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101948.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101948.log.gz 2026-03-23T19:58:52.804 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101993.log 2026-03-23T19:58:52.804 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101970.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.101970.log.gz 2026-03-23T19:58:52.808 INFO:teuthology.orchestra.run.vm08.stderr: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.101498.log.gz 2026-03-23T19:58:52.808 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102027.log 2026-03-23T19:58:52.809 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.101993.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101993.log.gz 2026-03-23T19:58:52.809 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102054.log 2026-03-23T19:58:52.810 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102027.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102027.log.gz 2026-03-23T19:58:52.810 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102087.log 2026-03-23T19:58:52.811 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102054.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102054.log.gz 2026-03-23T19:58:52.811 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102107.log 2026-03-23T19:58:52.813 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102087.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102087.log.gz 2026-03-23T19:58:52.813 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102130.log 2026-03-23T19:58:52.813 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102150.log 2026-03-23T19:58:52.814 INFO:teuthology.orchestra.run.vm08.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.102107.log.gz 2026-03-23T19:58:52.815 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102130.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102130.log.gz 2026-03-23T19:58:52.815 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102174.log 2026-03-23T19:58:52.816 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102191.log 2026-03-23T19:58:52.817 INFO:teuthology.orchestra.run.vm08.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.102150.log.gz 2026-03-23T19:58:52.817 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102174.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.102174.log.gz 2026-03-23T19:58:52.818 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102211.log 2026-03-23T19:58:52.819 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102191.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.102191.log.gz 2026-03-23T19:58:52.819 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102232.log 2026-03-23T19:58:52.819 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102254.log 2026-03-23T19:58:52.820 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.102211.log.gz 2026-03-23T19:58:52.820 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102232.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.102232.log.gz 2026-03-23T19:58:52.821 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102281.log 2026-03-23T19:58:52.821 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102254.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102307.log 2026-03-23T19:58:52.821 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.102254.log.gz 2026-03-23T19:58:52.822 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102281.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.102281.log.gz 2026-03-23T19:58:52.822 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102341.log 2026-03-23T19:58:52.823 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102307.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.102307.log.gz 2026-03-23T19:58:52.823 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102368.log 2026-03-23T19:58:52.824 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102341.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.102341.log.gz 2026-03-23T19:58:52.824 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102394.log 2026-03-23T19:58:52.825 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102368.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.102368.log.gz 2026-03-23T19:58:52.825 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102415.log 2026-03-23T19:58:52.826 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102394.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102394.log.gz 2026-03-23T19:58:52.826 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102448.log 2026-03-23T19:58:52.827 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102481.log 2026-03-23T19:58:52.827 INFO:teuthology.orchestra.run.vm08.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102415.log.gz 2026-03-23T19:58:52.828 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102448.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102448.log.gz 2026-03-23T19:58:52.828 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102508.log 2026-03-23T19:58:52.829 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102481.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102481.log.gz 2026-03-23T19:58:52.829 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102535.log 2026-03-23T19:58:52.829 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102508.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.102508.log.gz 2026-03-23T19:58:52.830 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102556.log 2026-03-23T19:58:52.830 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102535.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.102535.log.gz 2026-03-23T19:58:52.831 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102577.log 2026-03-23T19:58:52.831 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102556.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.102556.log.gz 2026-03-23T19:58:52.831 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102599.log 2026-03-23T19:58:52.832 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102577.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102621.log 2026-03-23T19:58:52.832 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102643.log 2026-03-23T19:58:52.833 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.102670.log 2026-03-23T19:58:52.833 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102697.log 2026-03-23T19:58:52.833 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102717.log 2026-03-23T19:58:52.834 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.102742.log 2026-03-23T19:58:52.834 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.102751.log 2026-03-23T19:58:52.837 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.102756.log 2026-03-23T19:58:52.837 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.102795.log 2026-03-23T19:58:52.838 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.102806.log 2026-03-23T19:58:52.839 INFO:teuthology.orchestra.run.vm08.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102577.log.gz 2026-03-23T19:58:52.839 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.102621.log: /var/log/ceph/cluster1-client.admin.102643.log: /var/log/ceph/cluster1-client.admin.102670.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.102670.log.gz 2026-03-23T19:58:52.840 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.102697.log: /var/log/ceph/cluster2-client.mirror.102717.log: /var/log/ceph/cluster2-client.mirror.0.102742.log: 18.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.102742.log.gz 2026-03-23T19:58:52.842 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.0.102751.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.102643.log.gz 2026-03-23T19:58:52.842 INFO:teuthology.orchestra.run.vm08.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.admin.102621.log.gz 2026-03-23T19:58:52.842 INFO:teuthology.orchestra.run.vm08.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.102697.log.gz 2026-03-23T19:58:52.843 INFO:teuthology.orchestra.run.vm08.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.102717.log.gz 2026-03-23T19:58:52.845 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.1.102756.log: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.102756.log.gz 2026-03-23T19:58:52.850 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.1.102795.log: /var/log/ceph/cluster1-client.admin.102599.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.102599.log.gz 2026-03-23T19:58:52.853 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.102839.log 2026-03-23T19:58:52.853 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.2.102806.log: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.102806.log.gz 2026-03-23T19:58:52.868 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.102868.log 2026-03-23T19:58:52.871 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.2.102839.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.102795.log.gz 2026-03-23T19:58:52.873 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.102933.log 2026-03-23T19:58:52.873 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.3.102868.log: 19.0%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.3.102868.log.gz 2026-03-23T19:58:52.873 INFO:teuthology.orchestra.run.vm08.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102954.log 2026-03-23T19:58:52.883 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.3.102933.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.2.102839.log.gz 2026-03-23T19:58:52.890 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103030.log 2026-03-23T19:58:52.890 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.102954.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102954.log.gz 2026-03-23T19:58:52.896 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103127.log 2026-03-23T19:58:52.896 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103030.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103030.log.gz 2026-03-23T19:58:52.897 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.3.102933.log.gz 2026-03-23T19:58:52.903 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103197.log 2026-03-23T19:58:52.903 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103127.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103127.log.gz 2026-03-23T19:58:52.910 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103222.log 2026-03-23T19:58:52.910 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103197.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103197.log.gz 2026-03-23T19:58:52.912 INFO:teuthology.orchestra.run.vm08.stderr: 95.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.102751.log.gz 2026-03-23T19:58:52.913 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103247.log 2026-03-23T19:58:52.913 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103222.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103222.log.gz 2026-03-23T19:58:52.913 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103272.log 2026-03-23T19:58:52.914 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103247.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103247.log.gz 2026-03-23T19:58:52.914 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103297.log 2026-03-23T19:58:52.914 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103272.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103272.log.gz 2026-03-23T19:58:52.915 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.103321.log 2026-03-23T19:58:52.915 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103297.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103297.log.gz 2026-03-23T19:58:52.915 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.103346.log 2026-03-23T19:58:52.915 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.103321.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.103321.log.gz 2026-03-23T19:58:52.916 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103374.log 2026-03-23T19:58:52.916 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.103346.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.103346.log.gz 2026-03-23T19:58:52.917 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103391.log 2026-03-23T19:58:52.917 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103374.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103374.log.gz 2026-03-23T19:58:52.917 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103411.log 2026-03-23T19:58:52.918 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103391.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.103391.log.gz 2026-03-23T19:58:52.918 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103431.log 2026-03-23T19:58:52.919 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103411.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103411.log.gz 2026-03-23T19:58:52.919 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103494.log 2026-03-23T19:58:52.920 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103520.log 2026-03-23T19:58:52.924 INFO:teuthology.orchestra.run.vm08.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103431.log.gz 2026-03-23T19:58:52.925 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103494.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103494.log.gz 2026-03-23T19:58:52.925 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103544.log 2026-03-23T19:58:52.925 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103520.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103520.log.gz 2026-03-23T19:58:52.926 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103568.log 2026-03-23T19:58:52.926 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103544.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.103544.log.gz 2026-03-23T19:58:52.927 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103592.log 2026-03-23T19:58:52.927 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103568.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.103568.log.gz 2026-03-23T19:58:52.927 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.103616.log 2026-03-23T19:58:52.928 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103592.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.103592.log.gz 2026-03-23T19:58:52.928 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.103643.log 2026-03-23T19:58:52.929 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.103616.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.103616.log.gz 2026-03-23T19:58:52.929 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.103670.log 2026-03-23T19:58:52.929 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.103643.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.103643.log.gz 2026-03-23T19:58:52.930 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.103697.log 2026-03-23T19:58:52.930 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.103670.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.103670.log.gz 2026-03-23T19:58:52.931 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103723.log 2026-03-23T19:58:52.931 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.103697.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.103697.log.gz 2026-03-23T19:58:52.931 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103780.log 2026-03-23T19:58:52.936 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103801.log 2026-03-23T19:58:52.937 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103780.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103780.log.gz 2026-03-23T19:58:52.943 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103823.log 2026-03-23T19:58:52.946 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.103801.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103801.log.gz 2026-03-23T19:58:52.955 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103933.log 2026-03-23T19:58:52.957 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103823.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103823.log.gz 2026-03-23T19:58:52.962 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.103955.log 2026-03-23T19:58:52.964 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.103933.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.103933.log.gz 2026-03-23T19:58:52.969 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.103964.log 2026-03-23T19:58:52.970 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.103955.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.103955.log.gz 2026-03-23T19:58:52.976 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.103975.log 2026-03-23T19:58:52.989 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.103964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.103992.log 2026-03-23T19:58:52.991 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.103975.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.103975.log.gz 2026-03-23T19:58:52.996 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.104002.log 2026-03-23T19:58:53.006 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.103992.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.104052.log 2026-03-23T19:58:53.006 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.104002.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.104002.log.gz 2026-03-23T19:58:53.016 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.104067.log 2026-03-23T19:58:53.026 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.104052.log: /var/log/ceph/cluster1-client.mirror.3.104067.log: gzip 19.6% -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.104125.log 2026-03-23T19:58:53.026 INFO:teuthology.orchestra.run.vm08.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.3.104067.log.gz 2026-03-23T19:58:53.028 INFO:teuthology.orchestra.run.vm08.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.104052.log.gz 2026-03-23T19:58:53.040 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.103723.log.gz 2026-03-23T19:58:53.042 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104160.log 2026-03-23T19:58:53.051 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.104125.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.104125.log.gz 2026-03-23T19:58:53.054 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104359.log 2026-03-23T19:58:53.056 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.104160.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.104160.log.gz 2026-03-23T19:58:53.058 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104385.log 2026-03-23T19:58:53.059 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104359.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104359.log.gz 2026-03-23T19:58:53.063 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104409.log 2026-03-23T19:58:53.067 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104385.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.104385.log.gz 2026-03-23T19:58:53.068 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104433.log 2026-03-23T19:58:53.071 INFO:teuthology.orchestra.run.vm08.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.103964.log.gz 2026-03-23T19:58:53.072 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104409.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.104409.log.gz 2026-03-23T19:58:53.072 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104457.log 2026-03-23T19:58:53.073 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104433.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.104433.log.gz 2026-03-23T19:58:53.073 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104481.log 2026-03-23T19:58:53.074 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104457.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.104457.log.gz 2026-03-23T19:58:53.074 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104505.log 2026-03-23T19:58:53.074 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104481.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104481.log.gz 2026-03-23T19:58:53.075 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104529.log 2026-03-23T19:58:53.075 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104505.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.104505.log.gz 2026-03-23T19:58:53.075 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104553.log 2026-03-23T19:58:53.076 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104529.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.104529.log.gz 2026-03-23T19:58:53.076 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104577.log 2026-03-23T19:58:53.076 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104553.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.104553.log.gz 2026-03-23T19:58:53.077 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104601.log 2026-03-23T19:58:53.077 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104577.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.104577.log.gz 2026-03-23T19:58:53.077 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104633.log 2026-03-23T19:58:53.078 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104601.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.104601.log.gz 2026-03-23T19:58:53.078 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104657.log 2026-03-23T19:58:53.078 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104633.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104633.log.gz 2026-03-23T19:58:53.079 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.104681.log 2026-03-23T19:58:53.079 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104657.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.104657.log.gz 2026-03-23T19:58:53.079 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.104708.log 2026-03-23T19:58:53.080 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.104681.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.104681.log.gz 2026-03-23T19:58:53.080 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.104735.log 2026-03-23T19:58:53.081 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.104708.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.104708.log.gz 2026-03-23T19:58:53.081 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.104762.log 2026-03-23T19:58:53.081 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.104735.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.104735.log.gz 2026-03-23T19:58:53.082 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104795.log 2026-03-23T19:58:53.082 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.104762.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.104762.log.gz 2026-03-23T19:58:53.082 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104819.log 2026-03-23T19:58:53.087 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.104795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104849.log 2026-03-23T19:58:53.103 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104819.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.104902.log 2026-03-23T19:58:53.113 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.104849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.104929.log 2026-03-23T19:58:53.113 INFO:teuthology.orchestra.run.vm08.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.104849.log.gz 2026-03-23T19:58:53.114 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.104902.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.104902.log.gz 2026-03-23T19:58:53.123 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.104956.log 2026-03-23T19:58:53.123 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.104929.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.104929.log.gz 2026-03-23T19:58:53.127 INFO:teuthology.orchestra.run.vm08.stderr: 95.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.103992.log.gz 2026-03-23T19:58:53.137 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.104983.log 2026-03-23T19:58:53.138 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.104956.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.104956.log.gz 2026-03-23T19:58:53.145 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105010.log 2026-03-23T19:58:53.146 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.104983.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.104983.log.gz 2026-03-23T19:58:53.152 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105030.log 2026-03-23T19:58:53.153 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105010.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.105010.log.gz 2026-03-23T19:58:53.160 INFO:teuthology.orchestra.run.vm08.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104795.log.gz 2026-03-23T19:58:53.160 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105057.log 2026-03-23T19:58:53.164 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105030.log: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.105030.log.gz 2026-03-23T19:58:53.164 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105074.log 2026-03-23T19:58:53.164 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105057.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105057.log.gz 2026-03-23T19:58:53.165 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105094.log 2026-03-23T19:58:53.165 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105074.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.105074.log.gz 2026-03-23T19:58:53.166 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105114.log 2026-03-23T19:58:53.166 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105094.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.105094.log.gz 2026-03-23T19:58:53.166 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105193.log 2026-03-23T19:58:53.168 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105114.log: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.104819.log.gz 2026-03-23T19:58:53.170 INFO:teuthology.orchestra.run.vm08.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.105114.log.gz 2026-03-23T19:58:53.170 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105219.log 2026-03-23T19:58:53.170 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105244.log 2026-03-23T19:58:53.171 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105219.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.105193.log.gz 2026-03-23T19:58:53.171 INFO:teuthology.orchestra.run.vm08.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.105219.log.gz 2026-03-23T19:58:53.171 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105268.log 2026-03-23T19:58:53.171 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105292.log 2026-03-23T19:58:53.172 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105244.log: /var/log/ceph/cluster1-client.mirror.105268.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105244.log.gz 2026-03-23T19:58:53.172 INFO:teuthology.orchestra.run.vm08.stderr: 87.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105316.log 2026-03-23T19:58:53.172 INFO:teuthology.orchestra.run.vm08.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.105268.log.gz 2026-03-23T19:58:53.172 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105292.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.105292.log.gz 2026-03-23T19:58:53.173 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105343.log 2026-03-23T19:58:53.173 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.105316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105370.log 2026-03-23T19:58:53.173 INFO:teuthology.orchestra.run.vm08.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.105316.log.gz 2026-03-23T19:58:53.173 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.105343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105397.log 2026-03-23T19:58:53.174 INFO:teuthology.orchestra.run.vm08.stderr: 87.3%/var/log/ceph/cluster2-client.admin.105370.log: -- replaced with /var/log/ceph/cluster1-client.admin.105343.log.gz 2026-03-23T19:58:53.174 INFO:teuthology.orchestra.run.vm08.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.105370.log.gz 2026-03-23T19:58:53.174 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105423.log 2026-03-23T19:58:53.174 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.105397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105444.log 2026-03-23T19:58:53.175 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105423.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.105397.log.gz 2026-03-23T19:58:53.175 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105464.log 2026-03-23T19:58:53.175 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105444.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.105444.log.gz 2026-03-23T19:58:53.176 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105489.log 2026-03-23T19:58:53.178 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105464.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105423.log.gz 2026-03-23T19:58:53.183 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105510.log 2026-03-23T19:58:53.184 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.105489.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.105489.log.gz 2026-03-23T19:58:53.189 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105620.log 2026-03-23T19:58:53.189 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105510.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105510.log.gz 2026-03-23T19:58:53.198 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105641.log 2026-03-23T19:58:53.198 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105620.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105620.log.gz 2026-03-23T19:58:53.204 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.105662.log 2026-03-23T19:58:53.211 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.105641.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105641.log.gz 2026-03-23T19:58:53.214 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.105671.log 2026-03-23T19:58:53.214 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.105662.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.105662.log.gz 2026-03-23T19:58:53.222 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.105679.log 2026-03-23T19:58:53.229 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.0.105671.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.105671.log.gz 2026-03-23T19:58:53.232 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.105699.log 2026-03-23T19:58:53.232 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.105679.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.105679.log.gz 2026-03-23T19:58:53.239 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.105707.log 2026-03-23T19:58:53.241 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.1.105699.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.105699.log.gz 2026-03-23T19:58:53.249 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.105760.log 2026-03-23T19:58:53.249 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.105707.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.105707.log.gz 2026-03-23T19:58:53.256 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.105769.log 2026-03-23T19:58:53.261 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.2.105760.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.105760.log.gz 2026-03-23T19:58:53.265 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.105837.log 2026-03-23T19:58:53.265 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.105769.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.105769.log.gz 2026-03-23T19:58:53.266 INFO:teuthology.orchestra.run.vm08.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.105464.log.gz 2026-03-23T19:58:53.266 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106193.log 2026-03-23T19:58:53.268 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.3.105837.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.105837.log.gz 2026-03-23T19:58:53.268 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106220.log 2026-03-23T19:58:53.269 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.106193.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.106193.log.gz 2026-03-23T19:58:53.269 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106247.log 2026-03-23T19:58:53.270 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.106220.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.106220.log.gz 2026-03-23T19:58:53.270 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106274.log 2026-03-23T19:58:53.271 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106247.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.106247.log.gz 2026-03-23T19:58:53.271 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106300.log 2026-03-23T19:58:53.271 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106274.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.106274.log.gz 2026-03-23T19:58:53.272 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106321.log 2026-03-23T19:58:53.275 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.106300.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.106300.log.gz 2026-03-23T19:58:53.275 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106343.log 2026-03-23T19:58:53.279 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.106321.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106321.log.gz 2026-03-23T19:58:53.279 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106363.log 2026-03-23T19:58:53.280 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.106343.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.106343.log.gz 2026-03-23T19:58:53.280 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106385.log 2026-03-23T19:58:53.284 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.mirror.106363.log: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.106363.log.gz 2026-03-23T19:58:53.284 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106405.log 2026-03-23T19:58:53.285 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.106385.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106385.log.gz 2026-03-23T19:58:53.285 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106426.log 2026-03-23T19:58:53.289 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.mirror.106405.log: 95.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106405.log.gz 2026-03-23T19:58:53.289 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106450.log 2026-03-23T19:58:53.289 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.106426.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.106426.log.gz 2026-03-23T19:58:53.290 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106474.log 2026-03-23T19:58:53.290 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106450.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106450.log.gz 2026-03-23T19:58:53.291 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106498.log 2026-03-23T19:58:53.291 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster1-client.admin.106474.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.106474.log.gz 2026-03-23T19:58:53.291 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106865.log 2026-03-23T19:58:53.292 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106498.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106498.log.gz 2026-03-23T19:58:53.292 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106914.log 2026-03-23T19:58:53.292 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106865.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106865.log.gz 2026-03-23T19:58:53.293 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106963.log 2026-03-23T19:58:53.293 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106914.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106914.log.gz 2026-03-23T19:58:53.293 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107012.log 2026-03-23T19:58:53.294 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.106963.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106963.log.gz 2026-03-23T19:58:53.294 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107061.log 2026-03-23T19:58:53.294 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107012.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107012.log.gz 2026-03-23T19:58:53.294 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107110.log 2026-03-23T19:58:53.295 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107061.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107061.log.gz 2026-03-23T19:58:53.295 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107159.log 2026-03-23T19:58:53.295 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107110.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107110.log.gz 2026-03-23T19:58:53.296 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107208.log 2026-03-23T19:58:53.296 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107159.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107159.log.gz 2026-03-23T19:58:53.296 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107257.log 2026-03-23T19:58:53.296 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107208.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107208.log.gz 2026-03-23T19:58:53.297 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107306.log 2026-03-23T19:58:53.297 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107257.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107257.log.gz 2026-03-23T19:58:53.297 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107355.log 2026-03-23T19:58:53.297 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107306.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107306.log.gz 2026-03-23T19:58:53.298 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107404.log 2026-03-23T19:58:53.298 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107355.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107355.log.gz 2026-03-23T19:58:53.298 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107451.log 2026-03-23T19:58:53.299 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107404.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107404.log.gz 2026-03-23T19:58:53.299 INFO:teuthology.orchestra.run.vm08.stderr:/var/log/ceph/cluster2-client.admin.107451.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107451.log.gz 2026-03-23T19:58:56.379 INFO:teuthology.orchestra.run.vm04.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-03-23T19:58:56.978 INFO:teuthology.orchestra.run.vm08.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-03-23T19:58:57.017 INFO:teuthology.orchestra.run.vm04.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-03-23T19:58:57.455 INFO:teuthology.orchestra.run.vm08.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-03-23T19:58:57.628 INFO:teuthology.orchestra.run.vm04.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-03-23T19:58:57.630 INFO:teuthology.orchestra.run.vm04.stderr: 2026-03-23T19:58:57.630 INFO:teuthology.orchestra.run.vm04.stderr:real 0m7.368s 2026-03-23T19:58:57.630 INFO:teuthology.orchestra.run.vm04.stderr:user 0m19.639s 2026-03-23T19:58:57.630 INFO:teuthology.orchestra.run.vm04.stderr:sys 0m1.547s 2026-03-23T19:58:58.022 INFO:teuthology.orchestra.run.vm08.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-03-23T19:58:58.026 INFO:teuthology.orchestra.run.vm08.stderr: 2026-03-23T19:58:58.026 INFO:teuthology.orchestra.run.vm08.stderr:real 0m7.736s 2026-03-23T19:58:58.026 INFO:teuthology.orchestra.run.vm08.stderr:user 0m20.986s 2026-03-23T19:58:58.026 INFO:teuthology.orchestra.run.vm08.stderr:sys 0m2.538s 2026-03-23T19:58:58.026 INFO:tasks.ceph:Archiving logs... 2026-03-23T19:58:58.026 DEBUG:teuthology.misc:Transferring archived files from vm04:/var/log/ceph to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm04/log 2026-03-23T19:58:58.027 DEBUG:teuthology.orchestra.run.vm04:> sudo tar c -f - -C /var/log/ceph -- . 2026-03-23T19:58:58.931 DEBUG:teuthology.misc:Transferring archived files from vm08:/var/log/ceph to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm08/log 2026-03-23T19:58:58.932 DEBUG:teuthology.orchestra.run.vm08:> sudo tar c -f - -C /var/log/ceph -- . 2026-03-23T19:59:00.082 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-03-23T19:59:00.084 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-03-23T19:59:00.084 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-03-23T19:59:00.086 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-03-23T19:59:00.151 INFO:teuthology.task.install.rpm:Removing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on rpm system. 2026-03-23T19:59:00.151 DEBUG:teuthology.orchestra.run.vm04:> 2026-03-23T19:59:00.151 DEBUG:teuthology.orchestra.run.vm04:> for d in ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd rbd-mirror ; do 2026-03-23T19:59:00.151 DEBUG:teuthology.orchestra.run.vm04:> sudo yum -y remove $d || true 2026-03-23T19:59:00.151 DEBUG:teuthology.orchestra.run.vm04:> done 2026-03-23T19:59:00.158 INFO:teuthology.task.install.rpm:Removing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on rpm system. 2026-03-23T19:59:00.158 DEBUG:teuthology.orchestra.run.vm08:> 2026-03-23T19:59:00.158 DEBUG:teuthology.orchestra.run.vm08:> for d in ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd rbd-mirror ; do 2026-03-23T19:59:00.158 DEBUG:teuthology.orchestra.run.vm08:> sudo yum -y remove $d || true 2026-03-23T19:59:00.158 DEBUG:teuthology.orchestra.run.vm08:> done 2026-03-23T19:59:00.344 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:00.344 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 103 M 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout: mailcap noarch 2.1.49-5.el9 @baseos 78 k 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Remove 2 Packages 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 103 M 2026-03-23T19:59:00.345 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:00.347 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:00.347 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:00.374 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:00.375 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:00.411 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-radosgw.target". 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-radosgw.target". 2026-03-23T19:59:00.436 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:00.447 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.470 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.487 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:00.562 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 103 M 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout: mailcap noarch 2.1.49-5.el9 @baseos 78 k 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Remove 2 Packages 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 103 M 2026-03-23T19:59:00.563 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:00.568 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:00.568 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:00.591 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:00.591 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:00.764 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-radosgw.target". 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-radosgw.target". 2026-03-23T19:59:00.791 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:00.795 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.876 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:00.892 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:00.947 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:00.947 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:01.066 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:01.066 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.067 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:01.067 INFO:teuthology.orchestra.run.vm04.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 mailcap-2.1.49-5.el9.noarch 2026-03-23T19:59:01.067 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.067 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:01.344 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:01.344 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:01.389 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:01.409 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:01.411 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:01.411 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 362 M 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: libxslt x86_64 1.1.34-12.el9 @appstream 743 k 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: socat x86_64 1.7.4.1-8.el9 @appstream 1.1 M 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: xmlstarlet x86_64 1.6.1-20.el9 @appstream 195 k 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Remove 4 Packages 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 364 M 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:01.412 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 2/2 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout: ceph-radosgw-2:20.2.0-712.g70f8415b.el9.x86_64 mailcap-2.1.49-5.el9.noarch 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.427 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:01.438 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:01.439 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 362 M 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: libxslt x86_64 1.1.34-12.el9 @appstream 743 k 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: socat x86_64 1.7.4.1-8.el9 @appstream 1.1 M 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: xmlstarlet x86_64 1.6.1-20.el9 @appstream 195 k 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout:Remove 4 Packages 2026-03-23T19:59:01.635 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.636 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 364 M 2026-03-23T19:59:01.636 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:01.637 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:01.639 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:01.639 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:01.645 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:01.648 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : xmlstarlet-1.6.1-20.el9.x86_64 2/4 2026-03-23T19:59:01.652 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libxslt-1.1.34-12.el9.x86_64 3/4 2026-03-23T19:59:01.664 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:01.664 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:01.671 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : socat-1.7.4.1-8.el9.x86_64 4/4 2026-03-23T19:59:01.726 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:01.733 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:01.735 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : xmlstarlet-1.6.1-20.el9.x86_64 2/4 2026-03-23T19:59:01.739 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libxslt-1.1.34-12.el9.x86_64 3/4 2026-03-23T19:59:01.742 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: socat-1.7.4.1-8.el9.x86_64 4/4 2026-03-23T19:59:01.742 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:01.742 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 2/4 2026-03-23T19:59:01.742 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 3/4 2026-03-23T19:59:01.756 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : socat-1.7.4.1-8.el9.x86_64 4/4 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 libxslt-1.1.34-12.el9.x86_64 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout: socat-1.7.4.1-8.el9.x86_64 xmlstarlet-1.6.1-20.el9.x86_64 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:01.795 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:01.824 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: socat-1.7.4.1-8.el9.x86_64 4/4 2026-03-23T19:59:01.825 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:01.825 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 2/4 2026-03-23T19:59:01.825 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 3/4 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout: ceph-test-2:20.2.0-712.g70f8415b.el9.x86_64 libxslt-1.1.34-12.el9.x86_64 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout: socat-1.7.4.1-8.el9.x86_64 xmlstarlet-1.6.1-20.el9.x86_64 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:01.876 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 0 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 6.8 M 2026-03-23T19:59:02.019 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 19 M 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: lua x86_64 5.4.4-4.el9 @appstream 593 k 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: lua-devel x86_64 5.4.4-4.el9 @crb 49 k 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: luarocks noarch 3.9.2-5.el9 @epel 692 k 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: unzip x86_64 6.0-59.el9 @baseos 389 k 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: zip x86_64 3.0-35.el9 @baseos 724 k 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout:Remove 8 Packages 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 28 M 2026-03-23T19:59:02.020 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:02.022 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:02.022 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:02.051 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:02.051 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:02.105 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:02.105 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: ceph x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 0 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 6.8 M 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 19 M 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: lua x86_64 5.4.4-4.el9 @appstream 593 k 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel x86_64 5.4.4-4.el9 @crb 49 k 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: luarocks noarch 3.9.2-5.el9 @epel 692 k 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: unzip x86_64 6.0-59.el9 @baseos 389 k 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: zip x86_64 3.0-35.el9 @baseos 724 k 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:Remove 8 Packages 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.106 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 28 M 2026-03-23T19:59:02.107 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:02.110 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:02.110 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:02.112 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/8 2026-03-23T19:59:02.116 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : luarocks-3.9.2-5.el9.noarch 2/8 2026-03-23T19:59:02.118 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : lua-devel-5.4.4-4.el9.x86_64 3/8 2026-03-23T19:59:02.121 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : zip-3.0-35.el9.x86_64 4/8 2026-03-23T19:59:02.124 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : unzip-6.0-59.el9.x86_64 5/8 2026-03-23T19:59:02.127 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : lua-5.4.4-4.el9.x86_64 6/8 2026-03-23T19:59:02.142 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:02.142 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:02.151 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.152 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:02.152 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-23T19:59:02.152 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mds.target". 2026-03-23T19:59:02.152 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mds.target". 2026-03-23T19:59:02.152 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.153 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.161 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mon.target". 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mon.target". 2026-03-23T19:59:02.189 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.190 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:02.192 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.196 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/8 2026-03-23T19:59:02.201 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : luarocks-3.9.2-5.el9.noarch 2/8 2026-03-23T19:59:02.203 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : lua-devel-5.4.4-4.el9.x86_64 3/8 2026-03-23T19:59:02.206 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : zip-3.0-35.el9.x86_64 4/8 2026-03-23T19:59:02.209 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : unzip-6.0-59.el9.x86_64 5/8 2026-03-23T19:59:02.211 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : lua-5.4.4-4.el9.x86_64 6/8 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mds.target". 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mds.target". 2026-03-23T19:59:02.232 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.233 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.244 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 7/8 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mon.target". 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mon.target". 2026-03-23T19:59:02.270 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.272 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 3/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lua-5.4.4-4.el9.x86_64 4/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 5/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 6/8 2026-03-23T19:59:02.286 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : unzip-6.0-59.el9.x86_64 7/8 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : zip-3.0-35.el9.x86_64 8/8 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: lua-5.4.4-4.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: unzip-6.0-59.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: zip-3.0-35.el9.x86_64 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.347 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 8/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-2:20.2.0-712.g70f8415b.el9.x86_64 1/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 3/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-5.4.4-4.el9.x86_64 4/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 5/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 6/8 2026-03-23T19:59:02.388 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : unzip-6.0-59.el9.x86_64 7/8 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : zip-3.0-35.el9.x86_64 8/8 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: ceph-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mds-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mon-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: lua-5.4.4-4.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: luarocks-3.9.2-5.el9.noarch 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: unzip-6.0-59.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: zip-3.0-35.el9.x86_64 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.445 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:02.590 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout:=========================================================================================== 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout:=========================================================================================== 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 24 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout:Removing dependent packages: 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 447 k 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.9 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 940 k 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 140 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 66 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 567 k 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 54 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.4 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 11 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: abseil-cpp x86_64 20211102.0-4.el9 @epel 1.9 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 98 M 2026-03-23T19:59:02.596 INFO:teuthology.orchestra.run.vm04.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 996 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.6 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 59 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 138 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: cryptsetup x86_64 2.8.1-3.el9 @baseos 770 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas x86_64 3.0.4-9.el9 @appstream 68 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 @appstream 11 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 @appstream 39 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: gperftools-libs x86_64 2.9.1-3.el9 @epel 1.4 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: grpc-data noarch 1.46.7-10.el9 @epel 13 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: ledmon-libs x86_64 1.1.0-3.el9 @baseos 80 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 409 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libconfig x86_64 1.7.2-9.el9 @baseos 220 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libgfortran x86_64 11.5.0-14.el9 @baseos 2.8 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: liboath x86_64 2.6.12-1.el9 @epel 94 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libquadmath x86_64 11.5.0-14.el9 @baseos 330 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 792 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libstoragemgmt x86_64 1.10.1-1.el9 @appstream 685 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: libunwind x86_64 1.6.2-1.el9 @epel 170 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: openblas x86_64 0.3.29-1.el9 @appstream 112 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: openblas-openmp x86_64 0.3.29-1.el9 @appstream 46 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: pciutils x86_64 3.7.0-7.el9 @baseos 216 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: protobuf x86_64 3.14.0-17.el9 @appstream 3.5 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-compiler x86_64 3.14.0-17.el9 @crb 2.9 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-asyncssh noarch 2.13.2-5.el9 @epel 3.9 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-autocommand noarch 2.2.2-8.el9 @epel 82 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-babel noarch 2.9.1-2.el9 @appstream 27 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 @epel 254 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-bcrypt x86_64 3.2.2-1.el9 @epel 87 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-cachetools noarch 4.2.4-1.el9 @epel 93 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 855 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-certifi noarch 2023.05.07-4.el9 @epel 6.3 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-cffi x86_64 1.14.5-5.el9 @baseos 1.0 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-chardet noarch 4.0.0-5.el9 @anaconda 1.4 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-cheroot noarch 10.0.1-4.el9 @epel 682 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-cherrypy noarch 18.6.1-2.el9 @epel 1.1 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-cryptography x86_64 36.0.1-5.el9 @baseos 4.5 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-devel x86_64 3.9.25-3.el9 @appstream 765 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-google-auth noarch 1:2.45.0-1.el9 @epel 1.4 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio x86_64 1.46.7-10.el9 @epel 6.7 M 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 @epel 418 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-idna noarch 2.10-7.el9.1 @anaconda 513 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco noarch 8.2.1-3.el9 @epel 3.7 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 @epel 24 k 2026-03-23T19:59:02.597 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 @epel 55 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-context noarch 6.0.1-3.el9 @epel 31 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 @epel 33 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-text noarch 4.0.0-2.el9 @epel 51 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jinja2 noarch 2.11.3-8.el9 @appstream 1.1 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jsonpatch noarch 1.21-16.el9 @koji-override-0 55 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-jsonpointer noarch 2.0-4.el9 @koji-override-0 34 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 @epel 21 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 @appstream 832 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-markupsafe x86_64 1.1.1-12.el9 @appstream 60 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-more-itertools noarch 8.12.0-2.el9 @epel 378 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-natsort noarch 7.1.1-5.el9 @epel 215 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy x86_64 1:1.23.5-2.el9 @appstream 30 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 @appstream 1.7 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-oauthlib noarch 3.1.1-5.el9 @koji-override-0 888 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-packaging noarch 20.9-5.el9 @appstream 248 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-ply noarch 3.11-14.el9 @baseos 430 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-portend noarch 3.1.0-2.el9 @epel 20 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-prettytable noarch 0.7.2-27.el9 @koji-override-0 166 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-protobuf noarch 3.14.0-17.el9 @appstream 1.4 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 @epel 389 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1 noarch 0.4.8-7.el9 @appstream 622 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 @appstream 1.0 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pycparser noarch 2.20-6.el9 @baseos 745 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyparsing noarch 2.4.7-9.el9 @baseos 635 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pysocks noarch 1.7.1-12.el9 @anaconda 88 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-pytz noarch 2021.1-5.el9 @koji-override-0 176 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-repoze-lru noarch 0.7-16.el9 @epel 83 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests noarch 2.25.1-10.el9 @baseos 405 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 @appstream 119 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-routes noarch 2.5.1-5.el9 @epel 459 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-rsa noarch 4.9-2.el9 @epel 202 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-scipy x86_64 1.9.3-2.el9 @appstream 76 M 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-tempora noarch 5.0.0-2.el9 @epel 96 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-toml noarch 0.10.2-6.el9 @appstream 99 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-typing-extensions noarch 4.15.0-1.el9 @epel 447 k 2026-03-23T19:59:02.598 INFO:teuthology.orchestra.run.vm04.stdout: python3-urllib3 noarch 1.26.5-7.el9 @baseos 746 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: python3-websocket-client noarch 1.2.3-2.el9 @epel 319 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: python3-zc-lockfile noarch 2.0-10.el9 @epel 35 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: qatlib x86_64 25.08.0-2.el9 @appstream 639 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-service x86_64 25.08.0-2.el9 @appstream 69 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: qatzip-libs x86_64 1.3.1-1.el9 @appstream 148 k 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout:=========================================================================================== 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout:Remove 98 Packages 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 666 M 2026-03-23T19:59:02.599 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:02.624 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:02.624 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:02.662 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout:=========================================================================================== 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout:=========================================================================================== 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 24 M 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout:Removing dependent packages: 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 447 k 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.9 M 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 940 k 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 140 M 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 66 M 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 567 k 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 54 M 2026-03-23T19:59:02.667 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.4 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 11 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp x86_64 20211102.0-4.el9 @epel 1.9 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 98 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 996 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.6 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 59 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 138 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup x86_64 2.8.1-3.el9 @baseos 770 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas x86_64 3.0.4-9.el9 @appstream 68 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 @appstream 11 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 @appstream 39 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs x86_64 2.9.1-3.el9 @epel 1.4 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data noarch 1.46.7-10.el9 @epel 13 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs x86_64 1.1.0-3.el9 @baseos 80 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 409 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libconfig x86_64 1.7.2-9.el9 @baseos 220 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran x86_64 11.5.0-14.el9 @baseos 2.8 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: liboath x86_64 2.6.12-1.el9 @epel 94 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath x86_64 11.5.0-14.el9 @baseos 330 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 792 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt x86_64 1.10.1-1.el9 @appstream 685 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: libunwind x86_64 1.6.2-1.el9 @epel 170 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: openblas x86_64 0.3.29-1.el9 @appstream 112 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp x86_64 0.3.29-1.el9 @appstream 46 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: pciutils x86_64 3.7.0-7.el9 @baseos 216 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: protobuf x86_64 3.14.0-17.el9 @appstream 3.5 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler x86_64 3.14.0-17.el9 @crb 2.9 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh noarch 2.13.2-5.el9 @epel 3.9 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand noarch 2.2.2-8.el9 @epel 82 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel noarch 2.9.1-2.el9 @appstream 27 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 @epel 254 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt x86_64 3.2.2-1.el9 @epel 87 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools noarch 4.2.4-1.el9 @epel 93 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 855 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi noarch 2023.05.07-4.el9 @epel 6.3 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi x86_64 1.14.5-5.el9 @baseos 1.0 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-chardet noarch 4.0.0-5.el9 @anaconda 1.4 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot noarch 10.0.1-4.el9 @epel 682 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy noarch 18.6.1-2.el9 @epel 1.1 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography x86_64 36.0.1-5.el9 @baseos 4.5 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel x86_64 3.9.25-3.el9 @appstream 765 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth noarch 1:2.45.0-1.el9 @epel 1.4 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio x86_64 1.46.7-10.el9 @epel 6.7 M 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 @epel 418 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-idna noarch 2.10-7.el9.1 @anaconda 513 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco noarch 8.2.1-3.el9 @epel 3.7 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 @epel 24 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 @epel 55 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context noarch 6.0.1-3.el9 @epel 31 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 @epel 33 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text noarch 4.0.0-2.el9 @epel 51 k 2026-03-23T19:59:02.668 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2 noarch 2.11.3-8.el9 @appstream 1.1 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-jsonpatch noarch 1.21-16.el9 @koji-override-0 55 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-jsonpointer noarch 2.0-4.el9 @koji-override-0 34 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 @epel 21 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 @appstream 832 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe x86_64 1.1.1-12.el9 @appstream 60 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools noarch 8.12.0-2.el9 @epel 378 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort noarch 7.1.1-5.el9 @epel 215 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy x86_64 1:1.23.5-2.el9 @appstream 30 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 @appstream 1.7 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-oauthlib noarch 3.1.1-5.el9 @koji-override-0 888 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging noarch 20.9-5.el9 @appstream 248 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply noarch 3.11-14.el9 @baseos 430 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend noarch 3.1.0-2.el9 @epel 20 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-prettytable noarch 0.7.2-27.el9 @koji-override-0 166 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf noarch 3.14.0-17.el9 @appstream 1.4 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 @epel 389 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1 noarch 0.4.8-7.el9 @appstream 622 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules noarch 0.4.8-7.el9 @appstream 1.0 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser noarch 2.20-6.el9 @baseos 745 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing noarch 2.4.7-9.el9 @baseos 635 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pysocks noarch 1.7.1-12.el9 @anaconda 88 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-pytz noarch 2021.1-5.el9 @koji-override-0 176 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru noarch 0.7-16.el9 @epel 83 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests noarch 2.25.1-10.el9 @baseos 405 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 @appstream 119 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes noarch 2.5.1-5.el9 @epel 459 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa noarch 4.9-2.el9 @epel 202 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy x86_64 1.9.3-2.el9 @appstream 76 M 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora noarch 5.0.0-2.el9 @epel 96 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml noarch 0.10.2-6.el9 @appstream 99 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions noarch 4.15.0-1.el9 @epel 447 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3 noarch 1.26.5-7.el9 @baseos 746 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client noarch 1.2.3-2.el9 @epel 319 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile noarch 2.0-10.el9 @epel 35 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: qatlib x86_64 25.08.0-2.el9 @appstream 639 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service x86_64 25.08.0-2.el9 @appstream 69 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs x86_64 1.3.1-1.el9 @appstream 148 k 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout:=========================================================================================== 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout:Remove 98 Packages 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:02.669 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 666 M 2026-03-23T19:59:02.670 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:02.696 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:02.697 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:02.755 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:02.756 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:02.821 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:02.822 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:02.911 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:02.912 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 1/98 2026-03-23T19:59:02.921 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 1/98 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mgr.target". 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mgr.target". 2026-03-23T19:59:02.945 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:02.946 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:02.959 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:02.995 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:02.995 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 1/98 2026-03-23T19:59:03.004 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 1/98 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-mgr.target". 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-mgr.target". 2026-03-23T19:59:03.024 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.025 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:03.041 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:03.045 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.n 3/98 2026-03-23T19:59:03.045 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 4/98 2026-03-23T19:59:03.101 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.n 3/98 2026-03-23T19:59:03.101 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 4/98 2026-03-23T19:59:03.119 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 4/98 2026-03-23T19:59:03.136 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-kubernetes-1:26.1.0-3.el9.noarch 5/98 2026-03-23T19:59:03.142 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-requests-oauthlib-1.3.0-12.el9.noarch 6/98 2026-03-23T19:59:03.142 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:03.157 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:03.165 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cherrypy-18.6.1-2.el9.noarch 8/98 2026-03-23T19:59:03.170 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cheroot-10.0.1-4.el9.noarch 9/98 2026-03-23T19:59:03.172 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 4/98 2026-03-23T19:59:03.181 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/98 2026-03-23T19:59:03.183 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-kubernetes-1:26.1.0-3.el9.noarch 5/98 2026-03-23T19:59:03.188 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-requests-oauthlib-1.3.0-12.el9.noarch 6/98 2026-03-23T19:59:03.188 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:03.191 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-grpcio-1.46.7-10.el9.x86_64 11/98 2026-03-23T19:59:03.202 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:03.209 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cherrypy-18.6.1-2.el9.noarch 8/98 2026-03-23T19:59:03.213 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cheroot-10.0.1-4.el9.noarch 9/98 2026-03-23T19:59:03.216 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.216 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.216 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-23T19:59:03.216 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-osd.target". 2026-03-23T19:59:03.217 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-osd.target". 2026-03-23T19:59:03.217 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:03.221 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.223 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/98 2026-03-23T19:59:03.228 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-grpcio-1.46.7-10.el9.x86_64 11/98 2026-03-23T19:59:03.231 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.249 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.249 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.249 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-23T19:59:03.249 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-osd.target". 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-osd.target". 2026-03-23T19:59:03.251 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.254 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.258 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.264 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:03.271 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.274 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/98 2026-03-23T19:59:03.281 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.281 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.281 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-03-23T19:59:03.281 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.281 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-text-4.0.0-2.el9.noarch 15/98 2026-03-23T19:59:03.286 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jinja2-2.11.3-8.el9.noarch 16/98 2026-03-23T19:59:03.290 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.297 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-requests-2.25.1-10.el9.noarch 17/98 2026-03-23T19:59:03.300 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 13/98 2026-03-23T19:59:03.303 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/98 2026-03-23T19:59:03.305 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-google-auth-1:2.45.0-1.el9.noarch 18/98 2026-03-23T19:59:03.310 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-text-4.0.0-2.el9.noarch 15/98 2026-03-23T19:59:03.316 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jinja2-2.11.3-8.el9.noarch 16/98 2026-03-23T19:59:03.319 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-rsa-4.9-2.el9.noarch 19/98 2026-03-23T19:59:03.326 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-requests-2.25.1-10.el9.noarch 17/98 2026-03-23T19:59:03.327 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pyasn1-modules-0.4.8-7.el9.noarch 20/98 2026-03-23T19:59:03.333 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-google-auth-1:2.45.0-1.el9.noarch 18/98 2026-03-23T19:59:03.345 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-rsa-4.9-2.el9.noarch 19/98 2026-03-23T19:59:03.353 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pyasn1-modules-0.4.8-7.el9.noarch 20/98 2026-03-23T19:59:03.372 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-urllib3-1.26.5-7.el9.noarch 21/98 2026-03-23T19:59:03.386 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-urllib3-1.26.5-7.el9.noarch 21/98 2026-03-23T19:59:03.390 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-babel-2.9.1-2.el9.noarch 22/98 2026-03-23T19:59:03.393 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-classes-3.2.1-5.el9.noarch 23/98 2026-03-23T19:59:03.394 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-babel-2.9.1-2.el9.noarch 22/98 2026-03-23T19:59:03.397 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-classes-3.2.1-5.el9.noarch 23/98 2026-03-23T19:59:03.404 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/98 2026-03-23T19:59:03.407 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/98 2026-03-23T19:59:03.415 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-asyncssh-2.13.2-5.el9.noarch 25/98 2026-03-23T19:59:03.415 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 26/98 2026-03-23T19:59:03.415 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-asyncssh-2.13.2-5.el9.noarch 25/98 2026-03-23T19:59:03.416 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 26/98 2026-03-23T19:59:03.423 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 26/98 2026-03-23T19:59:03.427 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 26/98 2026-03-23T19:59:03.527 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jsonpatch-1.21-16.el9.noarch 27/98 2026-03-23T19:59:03.541 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jsonpatch-1.21-16.el9.noarch 27/98 2026-03-23T19:59:03.545 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-scipy-1.9.3-2.el9.x86_64 28/98 2026-03-23T19:59:03.559 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.559 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/libstoragemgmt.service". 2026-03-23T19:59:03.559 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.561 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.583 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-scipy-1.9.3-2.el9.x86_64 28/98 2026-03-23T19:59:03.594 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.610 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.610 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/libstoragemgmt.service". 2026-03-23T19:59:03.610 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:03.611 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.613 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 30/98 2026-03-23T19:59:03.620 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cryptography-36.0.1-5.el9.x86_64 31/98 2026-03-23T19:59:03.623 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : protobuf-compiler-3.14.0-17.el9.x86_64 32/98 2026-03-23T19:59:03.625 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-bcrypt-3.2.2-1.el9.x86_64 33/98 2026-03-23T19:59:03.644 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target". 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target". 2026-03-23T19:59:03.645 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.646 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.657 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.660 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-context-6.0.1-3.el9.noarch 35/98 2026-03-23T19:59:03.663 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 30/98 2026-03-23T19:59:03.663 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-packaging-20.9-5.el9.noarch 36/98 2026-03-23T19:59:03.667 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-portend-3.1.0-2.el9.noarch 37/98 2026-03-23T19:59:03.670 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cryptography-36.0.1-5.el9.x86_64 31/98 2026-03-23T19:59:03.671 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-tempora-5.0.0-2.el9.noarch 38/98 2026-03-23T19:59:03.675 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-functools-3.5.0-2.el9.noarch 39/98 2026-03-23T19:59:03.676 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : protobuf-compiler-3.14.0-17.el9.x86_64 32/98 2026-03-23T19:59:03.680 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-routes-2.5.1-5.el9.noarch 40/98 2026-03-23T19:59:03.681 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-bcrypt-3.2.2-1.el9.x86_64 33/98 2026-03-23T19:59:03.686 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cffi-1.14.5-5.el9.x86_64 41/98 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target". 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target". 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:03.709 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.718 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 34/98 2026-03-23T19:59:03.722 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-context-6.0.1-3.el9.noarch 35/98 2026-03-23T19:59:03.725 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-packaging-20.9-5.el9.noarch 36/98 2026-03-23T19:59:03.728 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-portend-3.1.0-2.el9.noarch 37/98 2026-03-23T19:59:03.731 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-tempora-5.0.0-2.el9.noarch 38/98 2026-03-23T19:59:03.735 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-functools-3.5.0-2.el9.noarch 39/98 2026-03-23T19:59:03.737 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pycparser-2.20-6.el9.noarch 42/98 2026-03-23T19:59:03.740 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-routes-2.5.1-5.el9.noarch 40/98 2026-03-23T19:59:03.748 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cffi-1.14.5-5.el9.x86_64 41/98 2026-03-23T19:59:03.751 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-numpy-1:1.23.5-2.el9.x86_64 43/98 2026-03-23T19:59:03.755 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : flexiblas-netlib-3.0.4-9.el9.x86_64 44/98 2026-03-23T19:59:03.758 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 45/98 2026-03-23T19:59:03.760 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : openblas-openmp-0.3.29-1.el9.x86_64 46/98 2026-03-23T19:59:03.764 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libgfortran-11.5.0-14.el9.x86_64 47/98 2026-03-23T19:59:03.767 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 48/98 2026-03-23T19:59:03.790 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.790 INFO:teuthology.orchestra.run.vm08.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.790 INFO:teuthology.orchestra.run.vm08.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-23T19:59:03.790 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.791 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.800 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.802 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : openblas-0.3.29-1.el9.x86_64 50/98 2026-03-23T19:59:03.805 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : flexiblas-3.0.4-9.el9.x86_64 51/98 2026-03-23T19:59:03.808 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-ply-3.11-14.el9.noarch 52/98 2026-03-23T19:59:03.809 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pycparser-2.20-6.el9.noarch 42/98 2026-03-23T19:59:03.811 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-repoze-lru-0.7-16.el9.noarch 53/98 2026-03-23T19:59:03.813 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jaraco-8.2.1-3.el9.noarch 54/98 2026-03-23T19:59:03.816 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-more-itertools-8.12.0-2.el9.noarch 55/98 2026-03-23T19:59:03.818 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-toml-0.10.2-6.el9.noarch 56/98 2026-03-23T19:59:03.821 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pytz-2021.1-5.el9.noarch 57/98 2026-03-23T19:59:03.824 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pyparsing-2.4.7-9.el9.noarch 58/98 2026-03-23T19:59:03.833 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-backports-tarfile-1.2.0-1.el9.noarch 59/98 2026-03-23T19:59:03.837 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-numpy-1:1.23.5-2.el9.x86_64 43/98 2026-03-23T19:59:03.839 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-devel-3.9.25-3.el9.x86_64 60/98 2026-03-23T19:59:03.841 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-jsonpointer-2.0-4.el9.noarch 61/98 2026-03-23T19:59:03.844 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-typing-extensions-4.15.0-1.el9.noarch 62/98 2026-03-23T19:59:03.844 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : flexiblas-netlib-3.0.4-9.el9.x86_64 44/98 2026-03-23T19:59:03.847 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-idna-2.10-7.el9.1.noarch 63/98 2026-03-23T19:59:03.851 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 45/98 2026-03-23T19:59:03.853 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pysocks-1.7.1-12.el9.noarch 64/98 2026-03-23T19:59:03.855 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : openblas-openmp-0.3.29-1.el9.x86_64 46/98 2026-03-23T19:59:03.858 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-pyasn1-0.4.8-7.el9.noarch 65/98 2026-03-23T19:59:03.859 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libgfortran-11.5.0-14.el9.x86_64 47/98 2026-03-23T19:59:03.862 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 48/98 2026-03-23T19:59:03.864 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cachetools-4.2.4-1.el9.noarch 66/98 2026-03-23T19:59:03.868 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-chardet-4.0.0-5.el9.noarch 67/98 2026-03-23T19:59:03.871 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-autocommand-2.2.2-8.el9.noarch 68/98 2026-03-23T19:59:03.877 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : grpc-data-1.46.7-10.el9.noarch 69/98 2026-03-23T19:59:03.881 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-protobuf-3.14.0-17.el9.noarch 70/98 2026-03-23T19:59:03.884 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-zc-lockfile-2.0-10.el9.noarch 71/98 2026-03-23T19:59:03.892 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-natsort-7.1.1-5.el9.noarch 72/98 2026-03-23T19:59:03.897 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.897 INFO:teuthology.orchestra.run.vm04.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-03-23T19:59:03.897 INFO:teuthology.orchestra.run.vm04.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-03-23T19:59:03.897 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:03.898 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-oauthlib-3.1.1-5.el9.noarch 73/98 2026-03-23T19:59:03.898 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.901 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-websocket-client-1.2.3-2.el9.noarch 74/98 2026-03-23T19:59:03.904 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-certifi-2023.05.07-4.el9.noarch 75/98 2026-03-23T19:59:03.906 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9 76/98 2026-03-23T19:59:03.907 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9. 77/98 2026-03-23T19:59:03.908 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 49/98 2026-03-23T19:59:03.910 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : openblas-0.3.29-1.el9.x86_64 50/98 2026-03-23T19:59:03.912 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : flexiblas-3.0.4-9.el9.x86_64 51/98 2026-03-23T19:59:03.915 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-ply-3.11-14.el9.noarch 52/98 2026-03-23T19:59:03.917 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-repoze-lru-0.7-16.el9.noarch 53/98 2026-03-23T19:59:03.920 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jaraco-8.2.1-3.el9.noarch 54/98 2026-03-23T19:59:03.923 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-more-itertools-8.12.0-2.el9.noarch 55/98 2026-03-23T19:59:03.927 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:03.927 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2026-03-23T19:59:03.927 INFO:teuthology.orchestra.run.vm08.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-crash.service". 2026-03-23T19:59:03.927 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.928 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-toml-0.10.2-6.el9.noarch 56/98 2026-03-23T19:59:03.931 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pytz-2021.1-5.el9.noarch 57/98 2026-03-23T19:59:03.934 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pyparsing-2.4.7-9.el9.noarch 58/98 2026-03-23T19:59:03.935 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:03.935 INFO:teuthology.orchestra.run.vm08.stdout:warning: file /etc/logrotate.d/ceph: remove failed: No such file or directory 2026-03-23T19:59:03.935 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:03.943 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-backports-tarfile-1.2.0-1.el9.noarch 59/98 2026-03-23T19:59:03.948 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-devel-3.9.25-3.el9.x86_64 60/98 2026-03-23T19:59:03.950 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-jsonpointer-2.0-4.el9.noarch 61/98 2026-03-23T19:59:03.954 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-typing-extensions-4.15.0-1.el9.noarch 62/98 2026-03-23T19:59:03.958 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-idna-2.10-7.el9.1.noarch 63/98 2026-03-23T19:59:03.964 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pysocks-1.7.1-12.el9.noarch 64/98 2026-03-23T19:59:03.968 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-pyasn1-0.4.8-7.el9.noarch 65/98 2026-03-23T19:59:03.970 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:03.970 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 79/98 2026-03-23T19:59:03.977 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cachetools-4.2.4-1.el9.noarch 66/98 2026-03-23T19:59:03.980 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-chardet-4.0.0-5.el9.noarch 67/98 2026-03-23T19:59:03.983 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-autocommand-2.2.2-8.el9.noarch 68/98 2026-03-23T19:59:03.989 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : grpc-data-1.46.7-10.el9.noarch 69/98 2026-03-23T19:59:03.993 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-protobuf-3.14.0-17.el9.noarch 70/98 2026-03-23T19:59:03.997 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-zc-lockfile-2.0-10.el9.noarch 71/98 2026-03-23T19:59:04.009 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-natsort-7.1.1-5.el9.noarch 72/98 2026-03-23T19:59:04.015 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-oauthlib-3.1.1-5.el9.noarch 73/98 2026-03-23T19:59:04.018 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-websocket-client-1.2.3-2.el9.noarch 74/98 2026-03-23T19:59:04.021 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-certifi-2023.05.07-4.el9.noarch 75/98 2026-03-23T19:59:04.023 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9 76/98 2026-03-23T19:59:04.025 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9. 77/98 2026-03-23T19:59:04.044 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 79/98 2026-03-23T19:59:04.047 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:04.047 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2026-03-23T19:59:04.047 INFO:teuthology.orchestra.run.vm04.stdout:Removed "/etc/systemd/system/ceph.target.wants/ceph-crash.service". 2026-03-23T19:59:04.047 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:04.050 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : qatzip-libs-1.3.1-1.el9.x86_64 80/98 2026-03-23T19:59:04.054 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86 81/98 2026-03-23T19:59:04.056 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:04.056 INFO:teuthology.orchestra.run.vm04.stdout:warning: file /etc/logrotate.d/ceph: remove failed: No such file or directory 2026-03-23T19:59:04.056 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:04.056 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-prettytable-0.7.2-27.el9.noarch 82/98 2026-03-23T19:59:04.056 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 83/98 2026-03-23T19:59:04.097 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 78/98 2026-03-23T19:59:04.097 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 79/98 2026-03-23T19:59:04.132 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 79/98 2026-03-23T19:59:04.139 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : qatzip-libs-1.3.1-1.el9.x86_64 80/98 2026-03-23T19:59:04.142 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86 81/98 2026-03-23T19:59:04.145 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-prettytable-0.7.2-27.el9.noarch 82/98 2026-03-23T19:59:04.145 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 83/98 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 83/98 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /sys 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /proc 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /mnt 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /var/tmp 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /home 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /root 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout:skipping the directory /tmp 2026-03-23T19:59:09.954 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:09.965 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : qatlib-25.08.0-2.el9.x86_64 84/98 2026-03-23T19:59:09.985 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:09.985 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:09.995 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:09.999 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : gperftools-libs-2.9.1-3.el9.x86_64 86/98 2026-03-23T19:59:10.002 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libunwind-1.6.2-1.el9.x86_64 87/98 2026-03-23T19:59:10.004 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : pciutils-3.7.0-7.el9.x86_64 88/98 2026-03-23T19:59:10.007 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : liboath-2.6.12-1.el9.x86_64 89/98 2026-03-23T19:59:10.007 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 90/98 2026-03-23T19:59:10.022 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 90/98 2026-03-23T19:59:10.024 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ledmon-libs-1.1.0-3.el9.x86_64 91/98 2026-03-23T19:59:10.027 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libquadmath-11.5.0-14.el9.x86_64 92/98 2026-03-23T19:59:10.030 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : protobuf-3.14.0-17.el9.x86_64 93/98 2026-03-23T19:59:10.034 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libconfig-1.7.2-9.el9.x86_64 94/98 2026-03-23T19:59:10.040 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-markupsafe-1.1.1-12.el9.x86_64 95/98 2026-03-23T19:59:10.048 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : cryptsetup-2.8.1-3.el9.x86_64 96/98 2026-03-23T19:59:10.055 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : abseil-cpp-20211102.0-4.el9.x86_64 97/98 2026-03-23T19:59:10.055 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 83/98 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /sys 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /proc 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /mnt 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /var/tmp 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /home 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /root 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout:skipping the directory /tmp 2026-03-23T19:59:10.091 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.102 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : qatlib-25.08.0-2.el9.x86_64 84/98 2026-03-23T19:59:10.119 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:10.120 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:10.126 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 85/98 2026-03-23T19:59:10.129 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : gperftools-libs-2.9.1-3.el9.x86_64 86/98 2026-03-23T19:59:10.132 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libunwind-1.6.2-1.el9.x86_64 87/98 2026-03-23T19:59:10.135 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : pciutils-3.7.0-7.el9.x86_64 88/98 2026-03-23T19:59:10.137 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : liboath-2.6.12-1.el9.x86_64 89/98 2026-03-23T19:59:10.137 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 90/98 2026-03-23T19:59:10.150 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 90/98 2026-03-23T19:59:10.153 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ledmon-libs-1.1.0-3.el9.x86_64 91/98 2026-03-23T19:59:10.153 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 1/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9 4/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 5/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 6/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 8/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 9/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.n 10/98 2026-03-23T19:59:10.154 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 11/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9. 13/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 14/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 15/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 17/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 18/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 19/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 20/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 21/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 22/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 23/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 24/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 25/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 26/98 2026-03-23T19:59:10.155 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 27/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 28/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 31/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 32/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 33/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 34/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 35/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 36/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 37/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 39/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 40/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 41/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86 42/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 43/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 44/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-chardet-4.0.0-5.el9.noarch 45/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 46/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 47/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 49/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 50/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 51/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 52/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-idna-2.10-7.el9.1.noarch 53/98 2026-03-23T19:59:10.156 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libquadmath-11.5.0-14.el9.x86_64 92/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 54/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 55/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 56/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 57/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 58/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 59/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 60/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 61/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 62/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 63/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 64/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 65/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 66/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 67/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 68/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 69/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 70/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 71/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ply-3.11-14.el9.noarch 72/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 73/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 74/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 75/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 77/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 78/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 79/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 80/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pysocks-1.7.1-12.el9.noarch 81/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 82/98 2026-03-23T19:59:10.157 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 83/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 84/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 86/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 87/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 88/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 89/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 90/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 91/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 92/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 93/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 94/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 95/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 96/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 97/98 2026-03-23T19:59:10.158 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : protobuf-3.14.0-17.el9.x86_64 93/98 2026-03-23T19:59:10.162 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libconfig-1.7.2-9.el9.x86_64 94/98 2026-03-23T19:59:10.168 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-markupsafe-1.1.1-12.el9.x86_64 95/98 2026-03-23T19:59:10.177 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : cryptsetup-2.8.1-3.el9.x86_64 96/98 2026-03-23T19:59:10.182 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : abseil-cpp-20211102.0-4.el9.x86_64 97/98 2026-03-23T19:59:10.183 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.245 INFO:teuthology.orchestra.run.vm08.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-chardet-4.0.0-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-idna-2.10-7.el9.1.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jsonpatch-1.21-16.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-jsonpointer-2.0-4.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-ply-3.11-14.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-prettytable-0.7.2-27.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-23T19:59:10.246 INFO:teuthology.orchestra.run.vm08.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-pysocks-1.7.1-12.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-pytz-2021.1-5.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.247 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 1/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 3/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9 4/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-712.g70f8415b 5/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 6/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 7/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noar 8/98 2026-03-23T19:59:10.280 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-712.g70f841 9/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.n 10/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 11/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 12/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9. 13/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 14/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 15/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 17/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 18/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 19/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 20/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 21/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 22/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 23/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 24/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 25/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 26/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 27/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 28/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 29/98 2026-03-23T19:59:10.281 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 31/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 32/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 33/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 34/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 35/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 36/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 37/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 39/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 40/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 41/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86 42/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 43/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 44/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-chardet-4.0.0-5.el9.noarch 45/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 46/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 47/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 48/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 49/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 50/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 51/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 52/98 2026-03-23T19:59:10.282 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-idna-2.10-7.el9.1.noarch 53/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 54/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 55/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 56/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 57/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 58/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 59/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 60/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 61/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 62/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 63/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 64/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 65/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 66/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 67/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 68/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 69/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 70/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 71/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ply-3.11-14.el9.noarch 72/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 73/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 74/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 75/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyasn1-0.4.8-7.el9.noarch 77/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyasn1-modules-0.4.8-7.el9.noarch 78/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 79/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 80/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pysocks-1.7.1-12.el9.noarch 81/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 82/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 83/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 84/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 86/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 87/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 88/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 89/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 90/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 91/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 92/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 93/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 94/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 95/98 2026-03-23T19:59:10.284 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 96/98 2026-03-23T19:59:10.285 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 97/98 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 98/98 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: ceph-base-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: ceph-grafana-dashboards-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.360 INFO:teuthology.orchestra.run.vm04.stdout: ceph-immutable-object-cache-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-dashboard-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-diskprediction-local-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-modules-core-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-mgr-rook-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-osd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-prometheus-alerts-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-selinux-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ceph-volume-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: grpc-data-1.46.7-10.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libcephsqlite-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libconfig-1.7.2-9.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: liboath-2.6.12-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libradosstriper1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: libunwind-1.6.2-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: openblas-0.3.29-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: pciutils-3.7.0-7.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-3.14.0-17.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-babel-2.9.1-2.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-common-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-chardet-4.0.0-5.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-idna-2.10-7.el9.1.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-03-23T19:59:10.361 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jsonpatch-1.21-16.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-jsonpointer-2.0-4.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-packaging-20.9-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-ply-3.11-14.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-portend-3.1.0-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-prettytable-0.7.2-27.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-0.4.8-7.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyasn1-modules-0.4.8-7.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pycparser-2.20-6.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pysocks-1.7.1-12.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-pytz-2021.1-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-2.25.1-10.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-routes-2.5.1-5.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-rsa-4.9-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-toml-0.10.2-6.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-25.08.0-2.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: rbd-mirror-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.362 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.0 M 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Remove 1 Package 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 1.0 M 2026-03-23T19:59:10.462 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:10.464 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:10.464 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:10.465 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:10.465 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:10.483 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:10.483 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.592 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout: cephadm noarch 2:20.2.0-712.g70f8415b.el9 @ceph-noarch 1.0 M 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:Remove 1 Package 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 1.0 M 2026-03-23T19:59:10.593 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:10.595 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:10.595 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:10.596 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:10.596 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:10.600 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.613 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:10.613 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:10.641 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:10.767 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : cephadm-2:20.2.0-712.g70f8415b.el9.noarch 1/1 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout: cephadm-2:20.2.0-712.g70f8415b.el9.noarch 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-immutable-object-cache 2026-03-23T19:59:10.817 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:10.821 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:10.821 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:10.821 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.000 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-mgr 2026-03-23T19:59:11.000 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:11.003 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:11.004 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:11.004 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.011 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-immutable-object-cache 2026-03-23T19:59:11.011 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:11.014 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:11.015 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:11.015 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:11.271 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-mgr 2026-03-23T19:59:11.271 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:11.275 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:11.275 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:11.275 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:11.283 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-mgr-dashboard 2026-03-23T19:59:11.283 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:11.287 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:11.287 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:11.287 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.456 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-mgr-dashboard 2026-03-23T19:59:11.457 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:11.460 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:11.460 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:11.460 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:11.461 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-mgr-diskprediction-local 2026-03-23T19:59:11.462 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:11.465 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:11.465 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:11.465 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.641 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-mgr-rook 2026-03-23T19:59:11.642 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:11.645 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:11.645 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:11.645 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.785 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-mgr-diskprediction-local 2026-03-23T19:59:11.785 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:11.788 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:11.789 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:11.789 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:11.812 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-mgr-cephadm 2026-03-23T19:59:11.812 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:11.815 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:11.816 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:11.816 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:11.957 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-mgr-rook 2026-03-23T19:59:11.958 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:11.961 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:11.961 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:11.961 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:12.000 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.7 M 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout: fuse x86_64 2.9.9-17.el9 @baseos 214 k 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Remove 2 Packages 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 2.9 M 2026-03-23T19:59:12.001 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:12.003 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:12.003 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:12.017 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:12.018 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:12.047 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:12.051 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.065 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.144 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-mgr-cephadm 2026-03-23T19:59:12.144 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:12.147 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:12.147 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:12.147 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:12.148 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.148 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 fuse-2.9.9-17.el9.x86_64 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.187 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:12.353 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout: ceph-fuse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.7 M 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout: fuse x86_64 2.9.9-17.el9 @baseos 214 k 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Remove 2 Packages 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 2.9 M 2026-03-23T19:59:12.354 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:12.356 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:12.356 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:12.370 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:12.370 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:12.382 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: ceph-volume 2026-03-23T19:59:12.382 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:12.385 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:12.386 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:12.386 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:12.400 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:12.405 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.420 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.492 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.492 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.532 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : fuse-2.9.9-17.el9.x86_64 2/2 2026-03-23T19:59:12.532 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.532 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:12.533 INFO:teuthology.orchestra.run.vm04.stdout: ceph-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 fuse-2.9.9-17.el9.x86_64 2026-03-23T19:59:12.533 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.533 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:12.586 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repo Size 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 449 k 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Removing dependent packages: 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 155 k 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Remove 2 Packages 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 604 k 2026-03-23T19:59:12.587 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:12.589 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:12.589 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:12.600 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:12.600 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:12.631 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:12.633 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.649 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:12.713 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:12.713 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.716 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: ceph-volume 2026-03-23T19:59:12.716 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:12.719 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:12.720 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:12.752 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:12.752 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.752 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:12.752 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:12.753 INFO:teuthology.orchestra.run.vm08.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:12.753 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.753 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repo Size 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:12.904 INFO:teuthology.orchestra.run.vm04.stdout: librados-devel x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 449 k 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:Removing dependent packages: 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-devel x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 155 k 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:Remove 2 Packages 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 604 k 2026-03-23T19:59:12.905 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:12.907 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:12.918 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:12.918 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:12.936 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repo Size 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.4 M 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Removing dependent packages: 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 510 k 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 52 k 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 187 k 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Remove 4 Packages 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 3.2 M 2026-03-23T19:59:12.937 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:12.939 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:12.939 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:12.944 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:12.946 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:12.950 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:12.950 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:12.960 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:12.977 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:12.983 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:12.984 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86 2/4 2026-03-23T19:59:12.984 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 3/4 2026-03-23T19:59:12.997 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 3/4 2026-03-23T19:59:12.997 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.024 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:13.024 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 1/2 2026-03-23T19:59:13.056 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.056 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:13.056 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2/4 2026-03-23T19:59:13.056 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86 3/4 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2/2 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout: librados-devel-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.064 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.095 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repo Size 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 2.4 M 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Removing dependent packages: 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: python3-cephfs x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 510 k 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-proxy2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 52 k 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-argparse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 187 k 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout:Remove 4 Packages 2026-03-23T19:59:13.261 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.262 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 3.2 M 2026-03-23T19:59:13.262 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:13.263 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:13.270 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: libcephfs-devel 2026-03-23T19:59:13.271 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:13.274 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:13.275 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:13.275 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:13.275 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:13.275 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:13.422 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:13.425 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:13.427 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86 2/4 2026-03-23T19:59:13.427 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 3/4 2026-03-23T19:59:13.440 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 3/4 2026-03-23T19:59:13.441 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.504 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.505 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 1/4 2026-03-23T19:59:13.505 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2/4 2026-03-23T19:59:13.505 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86 3/4 2026-03-23T19:59:13.542 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 4/4 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs-proxy2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: libcephfs2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: python3-ceph-argparse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: python3-cephfs-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.543 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:13.577 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: Package Arch Version Repository Size 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:Removing: 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 12 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:Removing dependent packages: 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 1.1 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 1.1 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 264 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: qemu-kvm-block-rbd x86_64 17:10.1.0-16.el9 @appstream 37 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 238 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 498 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:Removing unused dependencies: 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options x86_64 1.75.0-13.el9 @appstream 276 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: libarrow x86_64 9.0.0-15.el9 @epel 18 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc noarch 9.0.0-15.el9 @epel 122 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: libnbd x86_64 1.20.3-4.el9 @appstream 453 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj x86_64 1.12.1-1.el9 @appstream 383 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq x86_64 0.11.0-7.el9 @appstream 102 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 10 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka x86_64 1.6.1-102.el9 @appstream 2.0 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 28 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust x86_64 2.12.0-6.el9 @appstream 1.0 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs x86_64 9.0.0-15.el9 @epel 2.8 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: re2 x86_64 1:20211101-20.el9 @epel 472 k 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: thrift x86_64 0.15.0-4.el9 @epel 4.8 M 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:Transaction Summary 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:================================================================================ 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout:Remove 20 Packages 2026-03-23T19:59:13.580 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.581 INFO:teuthology.orchestra.run.vm08.stdout:Freed space: 84 M 2026-03-23T19:59:13.581 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction check 2026-03-23T19:59:13.584 INFO:teuthology.orchestra.run.vm08.stdout:Transaction check succeeded. 2026-03-23T19:59:13.584 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction test 2026-03-23T19:59:13.607 INFO:teuthology.orchestra.run.vm08.stdout:Transaction test succeeded. 2026-03-23T19:59:13.607 INFO:teuthology.orchestra.run.vm08.stdout:Running transaction 2026-03-23T19:59:13.652 INFO:teuthology.orchestra.run.vm08.stdout: Preparing : 1/1 2026-03-23T19:59:13.655 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 1/20 2026-03-23T19:59:13.657 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2/20 2026-03-23T19:59:13.660 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 3/20 2026-03-23T19:59:13.660 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 4/20 2026-03-23T19:59:13.676 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 4/20 2026-03-23T19:59:13.678 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : parquet-libs-9.0.0-15.el9.x86_64 5/20 2026-03-23T19:59:13.680 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 6/20 2026-03-23T19:59:13.683 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 7/20 2026-03-23T19:59:13.684 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 8/20 2026-03-23T19:59:13.687 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libarrow-doc-9.0.0-15.el9.noarch 9/20 2026-03-23T19:59:13.687 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:13.702 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:13.702 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 11/20 2026-03-23T19:59:13.702 INFO:teuthology.orchestra.run.vm08.stdout:warning: file /etc/ceph: remove failed: No such file or directory 2026-03-23T19:59:13.702 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.719 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 11/20 2026-03-23T19:59:13.720 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: libcephfs-devel 2026-03-23T19:59:13.720 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:13.721 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libarrow-9.0.0-15.el9.x86_64 12/20 2026-03-23T19:59:13.723 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:13.724 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:13.724 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:13.725 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : re2-1:20211101-20.el9.x86_64 13/20 2026-03-23T19:59:13.728 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : lttng-ust-2.12.0-6.el9.x86_64 14/20 2026-03-23T19:59:13.731 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : thrift-0.15.0-4.el9.x86_64 15/20 2026-03-23T19:59:13.734 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libnbd-1.20.3-4.el9.x86_64 16/20 2026-03-23T19:59:13.736 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : libpmemobj-1.12.1-1.el9.x86_64 17/20 2026-03-23T19:59:13.738 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : boost-program-options-1.75.0-13.el9.x86_64 18/20 2026-03-23T19:59:13.741 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librabbitmq-0.11.0-7.el9.x86_64 19/20 2026-03-23T19:59:13.755 INFO:teuthology.orchestra.run.vm08.stdout: Erasing : librdkafka-1.6.1-102.el9.x86_64 20/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Running scriptlet: librdkafka-1.6.1-102.el9.x86_64 20/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 1/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 2/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 3/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 4/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 5/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 6/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 7/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 8/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 9/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 11/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 12/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 13/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 14/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 15/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 16/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 17/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 18/20 2026-03-23T19:59:13.826 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : re2-1:20211101-20.el9.x86_64 19/20 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 20/20 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout:Removed: 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: re2-1:20211101-20.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout: 2026-03-23T19:59:13.870 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:13.919 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: Package Arch Version Repository Size 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout:Removing: 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: librados2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 12 M 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout:Removing dependent packages: 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: python3-rados x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 1.1 M 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: python3-rbd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 1.1 M 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: python3-rgw x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 264 k 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: qemu-kvm-block-rbd x86_64 17:10.1.0-16.el9 @appstream 37 k 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: rbd-fuse x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 238 k 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: rbd-nbd x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 498 k 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout:Removing unused dependencies: 2026-03-23T19:59:13.920 INFO:teuthology.orchestra.run.vm04.stdout: boost-program-options x86_64 1.75.0-13.el9 @appstream 276 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: libarrow x86_64 9.0.0-15.el9 @epel 18 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-doc noarch 9.0.0-15.el9 @epel 122 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: libnbd x86_64 1.20.3-4.el9 @appstream 453 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: libpmemobj x86_64 1.12.1-1.el9 @appstream 383 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: librabbitmq x86_64 0.11.0-7.el9 @appstream 102 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: librbd1 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 10 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: librdkafka x86_64 1.6.1-102.el9 @appstream 2.0 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: librgw2 x86_64 2:20.2.0-712.g70f8415b.el9 @ceph 28 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: lttng-ust x86_64 2.12.0-6.el9 @appstream 1.0 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: parquet-libs x86_64 9.0.0-15.el9 @epel 2.8 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: re2 x86_64 1:20211101-20.el9 @epel 472 k 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: thrift x86_64 0.15.0-4.el9 @epel 4.8 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout:Transaction Summary 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout:================================================================================ 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout:Remove 20 Packages 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout:Freed space: 84 M 2026-03-23T19:59:13.921 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction check 2026-03-23T19:59:13.925 INFO:teuthology.orchestra.run.vm04.stdout:Transaction check succeeded. 2026-03-23T19:59:13.925 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction test 2026-03-23T19:59:13.950 INFO:teuthology.orchestra.run.vm04.stdout:Transaction test succeeded. 2026-03-23T19:59:13.950 INFO:teuthology.orchestra.run.vm04.stdout:Running transaction 2026-03-23T19:59:13.998 INFO:teuthology.orchestra.run.vm04.stdout: Preparing : 1/1 2026-03-23T19:59:14.001 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 1/20 2026-03-23T19:59:14.003 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2/20 2026-03-23T19:59:14.007 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 3/20 2026-03-23T19:59:14.007 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 4/20 2026-03-23T19:59:14.025 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 4/20 2026-03-23T19:59:14.027 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : parquet-libs-9.0.0-15.el9.x86_64 5/20 2026-03-23T19:59:14.029 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 6/20 2026-03-23T19:59:14.030 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 7/20 2026-03-23T19:59:14.032 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 8/20 2026-03-23T19:59:14.035 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libarrow-doc-9.0.0-15.el9.noarch 9/20 2026-03-23T19:59:14.035 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:14.048 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:14.048 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 11/20 2026-03-23T19:59:14.048 INFO:teuthology.orchestra.run.vm04.stdout:warning: file /etc/ceph: remove failed: No such file or directory 2026-03-23T19:59:14.048 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:14.056 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: librbd1 2026-03-23T19:59:14.056 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.060 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.061 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.061 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:14.063 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 11/20 2026-03-23T19:59:14.065 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libarrow-9.0.0-15.el9.x86_64 12/20 2026-03-23T19:59:14.069 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : re2-1:20211101-20.el9.x86_64 13/20 2026-03-23T19:59:14.072 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : lttng-ust-2.12.0-6.el9.x86_64 14/20 2026-03-23T19:59:14.075 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : thrift-0.15.0-4.el9.x86_64 15/20 2026-03-23T19:59:14.078 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libnbd-1.20.3-4.el9.x86_64 16/20 2026-03-23T19:59:14.080 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : libpmemobj-1.12.1-1.el9.x86_64 17/20 2026-03-23T19:59:14.082 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : boost-program-options-1.75.0-13.el9.x86_64 18/20 2026-03-23T19:59:14.086 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librabbitmq-0.11.0-7.el9.x86_64 19/20 2026-03-23T19:59:14.106 INFO:teuthology.orchestra.run.vm04.stdout: Erasing : librdkafka-1.6.1-102.el9.x86_64 20/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Running scriptlet: librdkafka-1.6.1-102.el9.x86_64 20/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 1/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 2/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 3/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 4/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 5/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 6/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librados2-2:20.2.0-712.g70f8415b.el9.x86_64 7/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 8/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 9/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 10/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 11/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 12/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 13/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 14/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 15/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 16/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 17/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 18/20 2026-03-23T19:59:14.169 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : re2-1:20211101-20.el9.x86_64 19/20 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 20/20 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout:Removed: 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-9.0.0-15.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: libnbd-1.20.3-4.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: librados2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: librbd1-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.212 INFO:teuthology.orchestra.run.vm04.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: librgw2-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: python3-rados-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: python3-rbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: python3-rgw-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: qemu-kvm-block-rbd-17:10.1.0-16.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: rbd-fuse-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: rbd-nbd-2:20.2.0-712.g70f8415b.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: re2-1:20211101-20.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: thrift-0.15.0-4.el9.x86_64 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout: 2026-03-23T19:59:14.213 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:14.235 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: python3-rados 2026-03-23T19:59:14.236 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.239 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.239 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.239 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:14.400 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: librbd1 2026-03-23T19:59:14.400 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:14.403 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:14.404 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:14.404 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:14.412 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: python3-rgw 2026-03-23T19:59:14.413 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.416 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.416 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.416 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:14.574 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: python3-rados 2026-03-23T19:59:14.574 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:14.577 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:14.578 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:14.578 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:14.589 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: python3-cephfs 2026-03-23T19:59:14.589 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.592 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.593 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.593 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:14.742 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: python3-rgw 2026-03-23T19:59:14.742 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:14.745 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:14.745 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:14.746 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:14.765 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: python3-rbd 2026-03-23T19:59:14.765 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.768 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.769 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.769 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:14.922 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: python3-cephfs 2026-03-23T19:59:14.922 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:14.925 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:14.926 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:14.926 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:14.942 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: rbd-fuse 2026-03-23T19:59:14.942 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:14.946 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:14.947 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:14.947 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:15.091 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: python3-rbd 2026-03-23T19:59:15.092 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:15.094 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:15.095 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:15.095 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:15.117 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: rbd-mirror 2026-03-23T19:59:15.117 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:15.120 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:15.121 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:15.121 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:15.266 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: rbd-fuse 2026-03-23T19:59:15.267 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:15.269 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:15.270 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:15.270 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:15.290 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: rbd-nbd 2026-03-23T19:59:15.290 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:15.294 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:15.294 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:15.294 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:15.438 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: rbd-mirror 2026-03-23T19:59:15.438 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:15.442 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:15.442 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:15.442 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:15.462 INFO:teuthology.orchestra.run.vm08.stdout:No match for argument: rbd-mirror 2026-03-23T19:59:15.462 INFO:teuthology.orchestra.run.vm08.stderr:No packages marked for removal. 2026-03-23T19:59:15.465 INFO:teuthology.orchestra.run.vm08.stdout:Dependencies resolved. 2026-03-23T19:59:15.466 INFO:teuthology.orchestra.run.vm08.stdout:Nothing to do. 2026-03-23T19:59:15.466 INFO:teuthology.orchestra.run.vm08.stdout:Complete! 2026-03-23T19:59:15.487 DEBUG:teuthology.orchestra.run.vm08:> sudo yum clean all 2026-03-23T19:59:15.611 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: rbd-nbd 2026-03-23T19:59:15.611 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:15.615 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:15.616 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:15.616 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:15.617 INFO:teuthology.orchestra.run.vm08.stdout:56 files removed 2026-03-23T19:59:15.636 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f /etc/yum.repos.d/ceph.repo 2026-03-23T19:59:15.660 DEBUG:teuthology.orchestra.run.vm08:> sudo yum clean expire-cache 2026-03-23T19:59:15.782 INFO:teuthology.orchestra.run.vm04.stdout:No match for argument: rbd-mirror 2026-03-23T19:59:15.783 INFO:teuthology.orchestra.run.vm04.stderr:No packages marked for removal. 2026-03-23T19:59:15.785 INFO:teuthology.orchestra.run.vm04.stdout:Dependencies resolved. 2026-03-23T19:59:15.786 INFO:teuthology.orchestra.run.vm04.stdout:Nothing to do. 2026-03-23T19:59:15.786 INFO:teuthology.orchestra.run.vm04.stdout:Complete! 2026-03-23T19:59:15.807 DEBUG:teuthology.orchestra.run.vm04:> sudo yum clean all 2026-03-23T19:59:15.820 INFO:teuthology.orchestra.run.vm08.stdout:Cache was expired 2026-03-23T19:59:15.820 INFO:teuthology.orchestra.run.vm08.stdout:0 files removed 2026-03-23T19:59:15.838 DEBUG:teuthology.parallel:result is None 2026-03-23T19:59:15.939 INFO:teuthology.orchestra.run.vm04.stdout:56 files removed 2026-03-23T19:59:15.955 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -f /etc/yum.repos.d/ceph.repo 2026-03-23T19:59:15.977 DEBUG:teuthology.orchestra.run.vm04:> sudo yum clean expire-cache 2026-03-23T19:59:16.127 INFO:teuthology.orchestra.run.vm04.stdout:Cache was expired 2026-03-23T19:59:16.127 INFO:teuthology.orchestra.run.vm04.stdout:0 files removed 2026-03-23T19:59:16.146 DEBUG:teuthology.parallel:result is None 2026-03-23T19:59:16.146 INFO:teuthology.task.install:Removing ceph sources lists on ubuntu@vm04.local 2026-03-23T19:59:16.146 INFO:teuthology.task.install:Removing ceph sources lists on ubuntu@vm08.local 2026-03-23T19:59:16.146 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -f /etc/yum.repos.d/ceph.repo 2026-03-23T19:59:16.146 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f /etc/yum.repos.d/ceph.repo 2026-03-23T19:59:16.170 DEBUG:teuthology.orchestra.run.vm08:> sudo mv -f /etc/yum/pluginconf.d/priorities.conf.orig /etc/yum/pluginconf.d/priorities.conf 2026-03-23T19:59:16.172 DEBUG:teuthology.orchestra.run.vm04:> sudo mv -f /etc/yum/pluginconf.d/priorities.conf.orig /etc/yum/pluginconf.d/priorities.conf 2026-03-23T19:59:16.234 DEBUG:teuthology.parallel:result is None 2026-03-23T19:59:16.239 DEBUG:teuthology.parallel:result is None 2026-03-23T19:59:16.239 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-03-23T19:59:16.241 INFO:teuthology.task.clock:Checking final clock skew... 2026-03-23T19:59:16.241 DEBUG:teuthology.orchestra.run.vm04:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-23T19:59:16.281 DEBUG:teuthology.orchestra.run.vm08:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-03-23T19:59:16.295 INFO:teuthology.orchestra.run.vm04.stderr:bash: line 1: ntpq: command not found 2026-03-23T19:59:16.295 INFO:teuthology.orchestra.run.vm08.stderr:bash: line 1: ntpq: command not found 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:=============================================================================== 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:^* srv01-nc.securepod.org 2 7 377 100 +2575us[+2623us] +/- 21ms 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:^+ ntp2.kernfusion.at 2 6 377 41 -1736us[-1736us] +/- 24ms 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:^+ kronos.mailus.de 2 8 377 235 -928us[ -896us] +/- 57ms 2026-03-23T19:59:16.300 INFO:teuthology.orchestra.run.vm04.stdout:^- y.ns.gin.ntt.net 2 8 377 107 -187us[ -138us] +/- 101ms 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:=============================================================================== 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:^+ ntp2.kernfusion.at 2 8 377 232 -2349us[-2411us] +/- 25ms 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:^- y.ns.gin.ntt.net 2 6 377 39 -278us[ -286us] +/- 102ms 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:^+ kronos.mailus.de 2 8 377 232 -1022us[-1083us] +/- 57ms 2026-03-23T19:59:16.301 INFO:teuthology.orchestra.run.vm08.stdout:^* srv01-nc.securepod.org 2 6 377 39 +2464us[+2456us] +/- 21ms 2026-03-23T19:59:16.301 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-03-23T19:59:16.303 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-03-23T19:59:16.304 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-03-23T19:59:16.306 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-03-23T19:59:16.309 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-03-23T19:59:16.311 INFO:teuthology.task.internal:Duration was 2230.238743 seconds 2026-03-23T19:59:16.311 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-03-23T19:59:16.313 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-03-23T19:59:16.313 DEBUG:teuthology.orchestra.run.vm04:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-03-23T19:59:16.342 DEBUG:teuthology.orchestra.run.vm08:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-03-23T19:59:16.379 INFO:teuthology.orchestra.run.vm08.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-23T19:59:16.380 INFO:teuthology.orchestra.run.vm04.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-03-23T19:59:16.690 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-03-23T19:59:16.690 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@vm04.local 2026-03-23T19:59:16.690 DEBUG:teuthology.orchestra.run.vm04:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-03-23T19:59:16.711 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@vm08.local 2026-03-23T19:59:16.711 DEBUG:teuthology.orchestra.run.vm08:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-03-23T19:59:16.754 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-03-23T19:59:16.754 DEBUG:teuthology.orchestra.run.vm04:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-23T19:59:16.756 DEBUG:teuthology.orchestra.run.vm08:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-23T19:59:17.323 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-03-23T19:59:17.323 DEBUG:teuthology.orchestra.run.vm04:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-03-23T19:59:17.325 DEBUG:teuthology.orchestra.run.vm08:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-03-23T19:59:17.350 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-23T19:59:17.351 INFO:teuthology.orchestra.run.vm04.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-23T19:59:17.351 INFO:teuthology.orchestra.run.vm04.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-23T19:59:17.351 INFO:teuthology.orchestra.run.vm04.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-03-23T19:59:17.351 INFO:teuthology.orchestra.run.vm04.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-03-23T19:59:17.352 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-23T19:59:17.352 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-23T19:59:17.353 INFO:teuthology.orchestra.run.vm08.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-03-23T19:59:17.353 INFO:teuthology.orchestra.run.vm08.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-03-23T19:59:17.353 INFO:teuthology.orchestra.run.vm08.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-03-23T19:59:17.503 INFO:teuthology.orchestra.run.vm04.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 98.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-03-23T19:59:17.546 INFO:teuthology.orchestra.run.vm08.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 98.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-03-23T19:59:17.548 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-03-23T19:59:17.551 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-03-23T19:59:17.551 DEBUG:teuthology.orchestra.run.vm04:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-03-23T19:59:17.578 DEBUG:teuthology.orchestra.run.vm08:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-03-23T19:59:17.616 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-03-23T19:59:17.619 DEBUG:teuthology.orchestra.run.vm04:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-03-23T19:59:17.621 DEBUG:teuthology.orchestra.run.vm08:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-03-23T19:59:17.649 INFO:teuthology.orchestra.run.vm04.stdout:kernel.core_pattern = core 2026-03-23T19:59:17.687 INFO:teuthology.orchestra.run.vm08.stdout:kernel.core_pattern = core 2026-03-23T19:59:17.704 DEBUG:teuthology.orchestra.run.vm04:> test -e /home/ubuntu/cephtest/archive/coredump 2026-03-23T19:59:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:59:17.721 DEBUG:teuthology.orchestra.run.vm08:> test -e /home/ubuntu/cephtest/archive/coredump 2026-03-23T19:59:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-23T19:59:17.762 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-03-23T19:59:17.765 INFO:teuthology.task.internal:Transferring archived files... 2026-03-23T19:59:17.765 DEBUG:teuthology.misc:Transferring archived files from vm04:/home/ubuntu/cephtest/archive to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm04 2026-03-23T19:59:17.765 DEBUG:teuthology.orchestra.run.vm04:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-03-23T19:59:17.800 DEBUG:teuthology.misc:Transferring archived files from vm08:/home/ubuntu/cephtest/archive to /archive/kyr-2026-03-20_22:04:26-rbd-tentacle-none-default-vps/3508/remote/vm08 2026-03-23T19:59:17.800 DEBUG:teuthology.orchestra.run.vm08:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-03-23T19:59:17.834 INFO:teuthology.task.internal:Removing archive directory... 2026-03-23T19:59:17.835 DEBUG:teuthology.orchestra.run.vm04:> rm -rf -- /home/ubuntu/cephtest/archive 2026-03-23T19:59:17.842 DEBUG:teuthology.orchestra.run.vm08:> rm -rf -- /home/ubuntu/cephtest/archive 2026-03-23T19:59:17.892 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-03-23T19:59:17.895 INFO:teuthology.task.internal:Not uploading archives. 2026-03-23T19:59:17.896 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-03-23T19:59:17.899 INFO:teuthology.task.internal:Tidying up after the test... 2026-03-23T19:59:17.899 DEBUG:teuthology.orchestra.run.vm04:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-03-23T19:59:17.901 DEBUG:teuthology.orchestra.run.vm08:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-03-23T19:59:17.916 INFO:teuthology.orchestra.run.vm04.stdout: 8532146 0 drwxr-xr-x 2 ubuntu ubuntu 6 Mar 23 19:59 /home/ubuntu/cephtest 2026-03-23T19:59:17.952 INFO:teuthology.orchestra.run.vm08.stdout: 8532145 0 drwxr-xr-x 2 ubuntu ubuntu 6 Mar 23 19:59 /home/ubuntu/cephtest 2026-03-23T19:59:17.954 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-03-23T19:59:17.961 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock} duration: 2230.2387433052063 flavor: default owner: kyr success: true 2026-03-23T19:59:17.962 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-23T19:59:17.996 INFO:teuthology.run:pass